Merge pull request #164 from Shopify/bb/primary-key-with-line-breaks

Handle composite primary keys with line breaks

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Unreleased
 
+# 4.2.3 (Jul, 2024)
+* Fix check for warnings against PKs with line breaks
+
 # 4.2.2 (Jun, 2024)
 * Avoid using the INSTANT algorithm.
 

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.2.2)
+    lhm-shopify (4.2.3)
       retriable (>= 3.0.0)
 
 GEM

gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.2)
+    lhm-shopify (4.2.3)
       retriable (>= 3.0.0)
 
 GEM

gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.2)
+    lhm-shopify (4.2.3)
       retriable (>= 3.0.0)
 
 GEM

gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.2)
+    lhm-shopify (4.2.3)
       retriable (>= 3.0.0)
 
 GEM

lib/lhm/chunker.rb

@@ -23,7 +23,6 @@ def initialize(migration, connection = nil, options = {})
       @chunk_finder = ChunkFinder.new(migration, connection, options)
       @options = options
       @raise_on_warnings = options.fetch(:raise_on_warnings, false)
-      @pk_duplicate_warning_regexp ||= /Duplicate entry .+ for key '(#{@migration.destination_name}\.)?PRIMARY'/
       @verifier = options[:verifier]
       if @throttler = options[:throttler]
         @throttler.connection = @connection if @throttler.respond_to?(:connection=)
@@ -81,7 +80,7 @@ def execute
 
     def raise_on_non_pk_duplicate_warning
       @connection.select_all("SHOW WARNINGS", should_retry: true, log_prefix: LOG_PREFIX).each do |row|
-        next if row["Message"].match?(@pk_duplicate_warning_regexp)
+        next if row["Message"].start_with?("Duplicate entry") && row["Message"].match?(/for key '(#{@migration.destination_name}\.)?PRIMARY'\z/)
 
         m = "Unexpected warning found for inserted row: #{row["Message"]}"
         Lhm.logger.warn(m)

lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.2.2'
+  VERSION = '4.2.3'
 end

spec/fixtures/composite_primary_key_with_varchar_columns.ddl

@@ -0,0 +1,10 @@
+CREATE TABLE `composite_primary_key_with_varchar_columns` (
+  `id` bigint NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint NOT NULL,
+  `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `owner_id` bigint NOT NULL,
+  `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`),
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC

spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl

@@ -0,0 +1,10 @@
+CREATE TABLE `composite_primary_key_with_varchar_columns_dest` (
+  `id` bigint NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint NOT NULL,
+  `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `owner_id` bigint NOT NULL,
+  `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`),
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC

spec/integration/chunker_spec.rb

@@ -61,6 +61,46 @@ def log_messages
       end
     end
 
+    it 'should copy and ignore duplicate composite primary key with line breaks' do
+      origin = table_create(:composite_primary_key_with_varchar_columns)
+      destination = table_create(:composite_primary_key_with_varchar_columns_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into composite_primary_key_with_varchar_columns set id = 1001, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  23
+', `key` = '
+  14
+
+  1
+'")
+      execute("insert into composite_primary_key_with_varchar_columns set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  22
+', `key` = '
+  14
+
+  1
+'")
+      execute("insert into composite_primary_key_with_varchar_columns_dest set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  22
+', `key` = '
+  14
+
+  1
+'")
+
+      Lhm::Chunker.new(migration, connection, {raise_on_warning: true, throttler: throttler, printer: printer} ).run
+
+      replica do
+        value(count_all(destination.name)).must_equal(2)
+      end
+    end
+
     it 'should copy and raise on unexpected warnings' do
       origin = table_create(:custom_primary_key)
       destination = table_create(:custom_primary_key_dest)