From d7d9dfd8d754042b4233381d916149a95db8ce9d Mon Sep 17 00:00:00 2001 From: Bastian Bartmann Date: Tue, 23 Jul 2024 09:07:12 +0200 Subject: [PATCH 1/2] Fix matching of primary key warning --- lib/lhm/chunker.rb | 3 +- ...osite_primary_key_with_varchar_columns.ddl | 10 +++++ ..._primary_key_with_varchar_columns_dest.ddl | 10 +++++ spec/integration/chunker_spec.rb | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/composite_primary_key_with_varchar_columns.ddl create mode 100644 spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl diff --git a/lib/lhm/chunker.rb b/lib/lhm/chunker.rb index 26146cf3..35f2d214 100644 --- a/lib/lhm/chunker.rb +++ b/lib/lhm/chunker.rb @@ -23,7 +23,6 @@ def initialize(migration, connection = nil, options = {}) @chunk_finder = ChunkFinder.new(migration, connection, options) @options = options @raise_on_warnings = options.fetch(:raise_on_warnings, false) - @pk_duplicate_warning_regexp ||= /Duplicate entry .+ for key '(#{@migration.destination_name}\.)?PRIMARY'/ @verifier = options[:verifier] if @throttler = options[:throttler] @throttler.connection = @connection if @throttler.respond_to?(:connection=) @@ -81,7 +80,7 @@ def execute def raise_on_non_pk_duplicate_warning @connection.select_all("SHOW WARNINGS", should_retry: true, log_prefix: LOG_PREFIX).each do |row| - next if row["Message"].match?(@pk_duplicate_warning_regexp) + next if row["Message"].start_with?("Duplicate entry") && row["Message"].match?(/for key '(#{@migration.destination_name}\.)?PRIMARY'\z/) m = "Unexpected warning found for inserted row: #{row["Message"]}" Lhm.logger.warn(m) diff --git a/spec/fixtures/composite_primary_key_with_varchar_columns.ddl b/spec/fixtures/composite_primary_key_with_varchar_columns.ddl new file mode 100644 index 00000000..6c92b2ac --- /dev/null +++ b/spec/fixtures/composite_primary_key_with_varchar_columns.ddl @@ -0,0 +1,10 @@ +CREATE TABLE `composite_primary_key_with_varchar_columns` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `shop_id` bigint NOT NULL, + `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `owner_id` bigint NOT NULL, + `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC diff --git a/spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl b/spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl new file mode 100644 index 00000000..82bfaee7 --- /dev/null +++ b/spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl @@ -0,0 +1,10 @@ +CREATE TABLE `composite_primary_key_with_varchar_columns_dest` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `shop_id` bigint NOT NULL, + `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `owner_id` bigint NOT NULL, + `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC diff --git a/spec/integration/chunker_spec.rb b/spec/integration/chunker_spec.rb index 4180a74d..b98f3d9c 100644 --- a/spec/integration/chunker_spec.rb +++ b/spec/integration/chunker_spec.rb @@ -61,6 +61,46 @@ def log_messages end end + it 'should copy and ignore duplicate composite primary key with line breaks' do + origin = table_create(:composite_primary_key_with_varchar_columns) + destination = table_create(:composite_primary_key_with_varchar_columns_dest) + migration = Lhm::Migration.new(origin, destination) + + execute("insert into composite_primary_key_with_varchar_columns set id = 1001, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = ' + 23 + + 23 +', `key` = ' + 14 + + 1 +'") + execute("insert into composite_primary_key_with_varchar_columns set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = ' + 23 + + 22 +', `key` = ' + 14 + + 1 +'") + execute("insert into composite_primary_key_with_varchar_columns_dest set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = ' + 23 + + 22 +', `key` = ' + 14 + + 1 +'") + + Lhm::Chunker.new(migration, connection, {raise_on_warning: true, throttler: throttler, printer: printer} ).run + + replica do + value(count_all(destination.name)).must_equal(2) + end + end + it 'should copy and raise on unexpected warnings' do origin = table_create(:custom_primary_key) destination = table_create(:custom_primary_key_dest) From d753bbf2177fe8801d7042ece5a54a22fb882561 Mon Sep 17 00:00:00 2001 From: Bastian Bartmann Date: Tue, 23 Jul 2024 09:44:17 +0200 Subject: [PATCH 2/2] Bump version --- CHANGELOG.md | 3 +++ Gemfile.lock | 2 +- gemfiles/activerecord_6.1.gemfile.lock | 2 +- gemfiles/activerecord_7.0.gemfile.lock | 2 +- gemfiles/activerecord_7.1.gemfile.lock | 2 +- lib/lhm/version.rb | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a87256..10c1815b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Unreleased +# 4.2.3 (Jul, 2024) +* Fix check for warnings against PKs with line breaks + # 4.2.2 (Jun, 2024) * Avoid using the INSTANT algorithm. diff --git a/Gemfile.lock b/Gemfile.lock index aedb0088..825376ae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - lhm-shopify (4.2.2) + lhm-shopify (4.2.3) retriable (>= 3.0.0) GEM diff --git a/gemfiles/activerecord_6.1.gemfile.lock b/gemfiles/activerecord_6.1.gemfile.lock index 62e6b876..7ba456e0 100644 --- a/gemfiles/activerecord_6.1.gemfile.lock +++ b/gemfiles/activerecord_6.1.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - lhm-shopify (4.2.2) + lhm-shopify (4.2.3) retriable (>= 3.0.0) GEM diff --git a/gemfiles/activerecord_7.0.gemfile.lock b/gemfiles/activerecord_7.0.gemfile.lock index eaf8ba71..e6c6164c 100644 --- a/gemfiles/activerecord_7.0.gemfile.lock +++ b/gemfiles/activerecord_7.0.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - lhm-shopify (4.2.2) + lhm-shopify (4.2.3) retriable (>= 3.0.0) GEM diff --git a/gemfiles/activerecord_7.1.gemfile.lock b/gemfiles/activerecord_7.1.gemfile.lock index a65d3c05..e7a2153f 100644 --- a/gemfiles/activerecord_7.1.gemfile.lock +++ b/gemfiles/activerecord_7.1.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - lhm-shopify (4.2.2) + lhm-shopify (4.2.3) retriable (>= 3.0.0) GEM diff --git a/lib/lhm/version.rb b/lib/lhm/version.rb index 638bb0af..4501248c 100644 --- a/lib/lhm/version.rb +++ b/lib/lhm/version.rb @@ -2,5 +2,5 @@ # Schmidt module Lhm - VERSION = '4.2.2' + VERSION = '4.2.3' end