diff --git a/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Capture b/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Capture new file mode 100644 index 0000000000..5d2b06e7fc --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithemptypoll_890703": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5,"updated_at":"2025-02-13 12:15:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6,"updated_at":"2025-02-13 12:16:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7,"updated_at":"2025-02-13 12:17:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8,"updated_at":"2025-02-13 12:18:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9,"updated_at":"2025-02-13 12:19:00"} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithemptypoll_890703":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:19:00"],"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery b/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery new file mode 100644 index 0000000000..8f59d68b6e --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithemptypoll_890703", + "schema": "test", + "table": "capturewithemptypoll_890703", + "cursor": [ + "updated_at" + ] + }, + "resource_path": [ + "test_capturewithemptypoll_890703" + ], + "collection": { + "name": "acmeCo/test/test_capturewithemptypoll_890703", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithemptypoll_890703" + } + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithModifications-Capture b/source-mysql-batch/.snapshots/TestCaptureWithModifications-Capture new file mode 100644 index 0000000000..e48d49c0cc --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithModifications-Capture @@ -0,0 +1,25 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithmodifications_786099": 17 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Initial value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 5","id":5,"updated_at":"2025-02-13 12:05:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 6","id":6,"updated_at":"2025-02-13 12:06:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 7","id":7,"updated_at":"2025-02-13 12:07:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 8","id":8,"updated_at":"2025-02-13 12:08:00"} +{"_meta":{"polled":"","index":999},"data":"Initial value for row 9","id":9,"updated_at":"2025-02-13 12:09:00"} +{"_meta":{"polled":"","index":999},"data":"Modified value for row 3","id":3,"updated_at":"2025-02-13 12:15:00"} +{"_meta":{"polled":"","index":999},"data":"Modified value for row 7","id":7,"updated_at":"2025-02-13 12:16:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10,"updated_at":"2025-02-13 12:20:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 11","id":11,"updated_at":"2025-02-13 12:21:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12,"updated_at":"2025-02-13 12:22:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 13","id":13,"updated_at":"2025-02-13 12:23:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 14","id":14,"updated_at":"2025-02-13 12:24:00"} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithmodifications_786099":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:24:00"],"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithModifications-Discovery b/source-mysql-batch/.snapshots/TestCaptureWithModifications-Discovery new file mode 100644 index 0000000000..149cf2523a --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithModifications-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithmodifications_786099", + "schema": "test", + "table": "capturewithmodifications_786099", + "cursor": [ + "updated_at" + ] + }, + "resource_path": [ + "test_capturewithmodifications_786099" + ], + "collection": { + "name": "acmeCo/test/test_capturewithmodifications_786099", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithmodifications_786099" + } + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Capture b/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Capture new file mode 100644 index 0000000000..c9d1e933b9 --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Capture @@ -0,0 +1,15 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithnullcursor_662607": 7 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value with NULL cursor","id":0,"sort_col":null} +{"_meta":{"polled":"","index":999},"data":"Another NULL cursor","id":2,"sort_col":null} +{"_meta":{"polled":"","index":999},"data":"Third NULL cursor","id":4,"sort_col":null} +{"_meta":{"polled":"","index":999},"data":"Value with cursor 10","id":1,"sort_col":10} +{"_meta":{"polled":"","index":999},"data":"Value with cursor 20","id":3,"sort_col":20} +{"_meta":{"polled":"","index":999},"data":"Value with cursor 25","id":7,"sort_col":25} +{"_meta":{"polled":"","index":999},"data":"Final value cursor 30","id":9,"sort_col":30} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithnullcursor_662607":{"CursorNames":["sort_col"],"CursorValues":[30],"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Discovery b/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Discovery new file mode 100644 index 0000000000..e8c7678a5b --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithnullcursor_662607", + "schema": "test", + "table": "capturewithnullcursor_662607", + "cursor": [ + "sort_col" + ] + }, + "resource_path": [ + "test_capturewithnullcursor_662607" + ], + "collection": { + "name": "acmeCo/test/test_capturewithnullcursor_662607", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithnullcursor_662607" + } + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Capture b/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Capture new file mode 100644 index 0000000000..d8087a3702 --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Capture @@ -0,0 +1,20 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithtwocolumncursor_321285": 12 Documents +# ================================ +{"_meta":{"polled":"","index":999},"col1":1,"col2":1,"data":"Value for row 0","id":0} +{"_meta":{"polled":"","index":999},"col1":1,"col2":2,"data":"Value for row 1","id":1} +{"_meta":{"polled":"","index":999},"col1":1,"col2":3,"data":"Value for row 2","id":2} +{"_meta":{"polled":"","index":999},"col1":2,"col2":1,"data":"Value for row 3","id":3} +{"_meta":{"polled":"","index":999},"col1":2,"col2":2,"data":"Value for row 4","id":4} +{"_meta":{"polled":"","index":999},"col1":2,"col2":3,"data":"Value for row 5","id":5} +{"_meta":{"polled":"","index":999},"col1":3,"col2":1,"data":"Value for row 8","id":8} +{"_meta":{"polled":"","index":999},"col1":3,"col2":2,"data":"Value for row 9","id":9} +{"_meta":{"polled":"","index":999},"col1":3,"col2":3,"data":"Value for row 10","id":10} +{"_meta":{"polled":"","index":999},"col1":4,"col2":1,"data":"Value for row 11","id":11} +{"_meta":{"polled":"","index":999},"col1":4,"col2":2,"data":"Value for row 12","id":12} +{"_meta":{"polled":"","index":999},"col1":4,"col2":3,"data":"Value for row 13","id":13} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithtwocolumncursor_321285":{"CursorNames":["col1","col2"],"CursorValues":[4,3],"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery b/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery new file mode 100644 index 0000000000..19a8895e9b --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery @@ -0,0 +1,59 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithtwocolumncursor_321285", + "schema": "test", + "table": "capturewithtwocolumncursor_321285", + "cursor": [ + "col1", + "col2" + ] + }, + "resource_path": [ + "test_capturewithtwocolumncursor_321285" + ], + "collection": { + "name": "acmeCo/test/test_capturewithtwocolumncursor_321285", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithtwocolumncursor_321285" + } + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Capture b/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Capture new file mode 100644 index 0000000000..b46f73237b --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Capture @@ -0,0 +1,28 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithupdatedatcursor_792371": 20 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5,"updated_at":"2025-02-13 12:05:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6,"updated_at":"2025-02-13 12:06:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7,"updated_at":"2025-02-13 12:07:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8,"updated_at":"2025-02-13 12:08:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9,"updated_at":"2025-02-13 12:09:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10,"updated_at":"2025-02-13 12:10:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 11","id":11,"updated_at":"2025-02-13 12:11:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12,"updated_at":"2025-02-13 12:12:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 13","id":13,"updated_at":"2025-02-13 12:13:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 14","id":14,"updated_at":"2025-02-13 12:14:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 15","id":15,"updated_at":"2025-02-13 12:15:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 16","id":16,"updated_at":"2025-02-13 12:16:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 17","id":17,"updated_at":"2025-02-13 12:17:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 18","id":18,"updated_at":"2025-02-13 12:18:00"} +{"_meta":{"polled":"","index":999},"data":"Value for row 19","id":19,"updated_at":"2025-02-13 12:19:00"} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithupdatedatcursor_792371":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:19:00"],"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery b/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery new file mode 100644 index 0000000000..72856140d8 --- /dev/null +++ b/source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithupdatedatcursor_792371", + "schema": "test", + "table": "capturewithupdatedatcursor_792371", + "cursor": [ + "updated_at" + ] + }, + "resource_path": [ + "test_capturewithupdatedatcursor_792371" + ], + "collection": { + "name": "acmeCo/test/test_capturewithupdatedatcursor_792371", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithupdatedatcursor_792371" + } + diff --git a/source-mysql-batch/.snapshots/TestFullRefresh-Capture b/source-mysql-batch/.snapshots/TestFullRefresh-Capture new file mode 100644 index 0000000000..8912fb8247 --- /dev/null +++ b/source-mysql-batch/.snapshots/TestFullRefresh-Capture @@ -0,0 +1,38 @@ +# ================================ +# Collection "acmeCo/test/test_fullrefresh_902536": 30 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9} +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9} +{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10} +{"_meta":{"polled":"","index":999},"data":"Value for row 11","id":11} +{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12} +{"_meta":{"polled":"","index":999},"data":"Value for row 13","id":13} +{"_meta":{"polled":"","index":999},"data":"Value for row 14","id":14} +{"_meta":{"polled":"","index":999},"data":"Value for row 15","id":15} +{"_meta":{"polled":"","index":999},"data":"Value for row 16","id":16} +{"_meta":{"polled":"","index":999},"data":"Value for row 17","id":17} +{"_meta":{"polled":"","index":999},"data":"Value for row 18","id":18} +{"_meta":{"polled":"","index":999},"data":"Value for row 19","id":19} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_fullrefresh_902536":{"LastPolled":""}}} + diff --git a/source-mysql-batch/.snapshots/TestFullRefresh-Discovery b/source-mysql-batch/.snapshots/TestFullRefresh-Discovery new file mode 100644 index 0000000000..dd709c4f20 --- /dev/null +++ b/source-mysql-batch/.snapshots/TestFullRefresh-Discovery @@ -0,0 +1,55 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_fullrefresh_902536", + "schema": "test", + "table": "fullrefresh_902536" + }, + "resource_path": [ + "test_fullrefresh_902536" + ], + "collection": { + "name": "acmeCo/test/test_fullrefresh_902536", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_fullrefresh_902536" + } + diff --git a/source-mysql-batch/main_test.go b/source-mysql-batch/main_test.go index c4c35f9d93..aa3283fb59 100644 --- a/source-mysql-batch/main_test.go +++ b/source-mysql-batch/main_test.go @@ -542,3 +542,197 @@ func TestJSONType(t *testing.T) { cupaloy.SnapshotT(t, cs.Summary()) }) } + +// TestFullRefresh exercises the scenario of a table without a configured cursor, +// which causes the capture to perform a full refresh every time it runs. +func TestFullRefresh(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for i := 0; i < 10; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s VALUES (?, ?)", tableName), i, fmt.Sprintf("Value for row %d", i)) + } + cs.Capture(ctx, t, nil) + for i := 10; i < 20; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s VALUES (?, ?)", tableName), i, fmt.Sprintf("Value for row %d", i)) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestCaptureWithUpdatedAtCursor exercises the use-case of a capture using a non-primary-key updated_at column as the cursor. +func TestCaptureWithUpdatedAtCursor(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT, updated_at TIMESTAMP)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + setResourceCursor(t, cs.Bindings[0], "updated_at") + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + baseTime := time.Date(2025, 2, 13, 12, 0, 0, 0, time.UTC) + for i := 0; i < 10; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Value for row %d", i), baseTime.Add(time.Duration(i)*time.Minute).Format("2006-01-02 15:04:05")) + } + cs.Capture(ctx, t, nil) + for i := 10; i < 20; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Value for row %d", i), baseTime.Add(time.Duration(i)*time.Minute).Format("2006-01-02 15:04:05")) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestCaptureWithTwoColumnCursor exercises the use-case of a capture using a multiple-column compound cursor. +func TestCaptureWithTwoColumnCursor(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, col1 INTEGER, col2 INTEGER, data TEXT)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + setResourceCursor(t, cs.Bindings[0], "col1", "col2") + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + {0, 1, 1, "Value for row 0"}, {1, 1, 2, "Value for row 1"}, {2, 1, 3, "Value for row 2"}, + {3, 2, 1, "Value for row 3"}, {4, 2, 2, "Value for row 4"}, {5, 2, 3, "Value for row 5"}, + } { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, col1, col2, data) VALUES (?, ?, ?, ?)", tableName), row...) + } + cs.Capture(ctx, t, nil) + for _, row := range [][]any{ + {6, 0, 9, "Value ignored because col1 is too small"}, + {7, 2, 0, "Value ignored because col2 is too small"}, + {8, 3, 1, "Value for row 8"}, {9, 3, 2, "Value for row 9"}, {10, 3, 3, "Value for row 10"}, + {11, 4, 1, "Value for row 11"}, {12, 4, 2, "Value for row 12"}, {13, 4, 3, "Value for row 13"}, + } { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, col1, col2, data) VALUES (?, ?, ?, ?)", tableName), row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestCaptureWithModifications exercises the use-case of a capture using an updated_at +// cursor where some rows are modified and deleted between captures. +func TestCaptureWithModifications(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT, updated_at TIMESTAMP)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + setResourceCursor(t, cs.Bindings[0], "updated_at") + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + baseTime := time.Date(2025, 2, 13, 12, 0, 0, 0, time.UTC) + + for i := 0; i < 10; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Initial value for row %d", i), baseTime.Add(time.Duration(i)*time.Minute).Format("2006-01-02 15:04:05")) + } + cs.Capture(ctx, t, nil) + + // Update and delete some rows, as well as inserting a few more. + executeControlQuery(t, control, fmt.Sprintf("UPDATE %s SET data = ?, updated_at = ? WHERE id = ?", tableName), + "Modified value for row 3", baseTime.Add(15*time.Minute).Format("2006-01-02 15:04:05"), 3) + executeControlQuery(t, control, fmt.Sprintf("UPDATE %s SET data = ?, updated_at = ? WHERE id = ?", tableName), + "Modified value for row 7", baseTime.Add(16*time.Minute).Format("2006-01-02 15:04:05"), 7) + executeControlQuery(t, control, fmt.Sprintf("DELETE FROM %s WHERE id IN (2, 5)", tableName)) + for i := 10; i < 15; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Value for row %d", i), baseTime.Add(time.Duration(i+10)*time.Minute).Format("2006-01-02 15:04:05")) + } + + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestCaptureWithEmptyPoll exercises the scenario where a polling interval finds no new rows. +func TestCaptureWithEmptyPoll(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT, updated_at TIMESTAMP)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + setResourceCursor(t, cs.Bindings[0], "updated_at") + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + baseTime := time.Date(2025, 2, 13, 12, 0, 0, 0, time.UTC) + + // First batch of rows + for i := 0; i < 5; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Value for row %d", i), baseTime.Add(time.Duration(i)*time.Minute).Format("2006-01-02 15:04:05")) + } + cs.Capture(ctx, t, nil) + + // No changes + cs.Capture(ctx, t, nil) + + // Second batch of rows with later timestamps + for i := 5; i < 10; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, updated_at) VALUES (?, ?, ?)", tableName), + i, fmt.Sprintf("Value for row %d", i), baseTime.Add(time.Duration(i+10)*time.Minute).Format("2006-01-02 15:04:05")) + } + cs.Capture(ctx, t, nil) + + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestCaptureWithNullCursor exercises the handling of NULL values in cursor columns. +func TestCaptureWithNullCursor(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testMySQLClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT, sort_col INTEGER)") + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + setResourceCursor(t, cs.Bindings[0], "sort_col") + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + // First batch with mix of NULL and non-NULL cursor values + for _, row := range [][]any{ + {0, "Value with NULL cursor", nil}, + {1, "Value with cursor 10", 10}, + {2, "Another NULL cursor", nil}, + {3, "Value with cursor 20", 20}, + {4, "Third NULL cursor", nil}, + } { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, sort_col) VALUES (?, ?, ?)", tableName), row...) + } + cs.Capture(ctx, t, nil) + + // Second batch testing NULL handling after initial cursor + for _, row := range [][]any{ + {5, "Late NULL cursor", nil}, // Will not be captured + {6, "Value with cursor 15", 15}, // Will not be captured (cursor 20 is already seen) + {7, "Value with cursor 25", 25}, + {8, "Another late NULL", nil}, // Will not be captured + {9, "Final value cursor 30", 30}, + } { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, data, sort_col) VALUES (?, ?, ?)", tableName), row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +}