-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
source-mysql-batch: Exercise more cursor/key scenarios
Adds more test scenarios, including: - A full-refresh table with no cursor. - A polling interval which yields no results. - Modifications and deletions as well as inserts. - Null values in the cursor column. - A two-column compound cursor. - An explicit query template and no table/schema names, like what all preexisting captures in production have.
- Loading branch information
1 parent
b8e19fd
commit d78da40
Showing
15 changed files
with
803 additions
and
0 deletions.
There are no files selected for viewing
18 changes: 18 additions & 0 deletions
18
source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Capture
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test_capturewithemptypoll_890703": 10 Documents | ||
| # ================================ | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 5","id":5,"updated_at":"2025-02-13 12:15:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 6","id":6,"updated_at":"2025-02-13 12:16:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 7","id":7,"updated_at":"2025-02-13 12:17:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 8","id":8,"updated_at":"2025-02-13 12:18:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 9","id":9,"updated_at":"2025-02-13 12:19:00"} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test_capturewithemptypoll_890703":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:19:00"],"LastPolled":"<TIMESTAMP>"}}} | ||
|
|
58 changes: 58 additions & 0 deletions
58
source-mysql-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| Binding 0: | ||
| { | ||
| "resource_config_json": { | ||
| "name": "test_capturewithemptypoll_890703", | ||
| "schema": "test", | ||
| "table": "capturewithemptypoll_890703", | ||
| "cursor": [ | ||
| "updated_at" | ||
| ] | ||
| }, | ||
| "resource_path": [ | ||
| "test_capturewithemptypoll_890703" | ||
| ], | ||
| "collection": { | ||
| "name": "acmeCo/test/test_capturewithemptypoll_890703", | ||
| "read_schema_json": { | ||
| "type": "object", | ||
| "required": [ | ||
| "_meta", | ||
| "id" | ||
| ], | ||
| "properties": { | ||
| "_meta": { | ||
| "$schema": "http://json-schema.org/draft/2020-12/schema", | ||
| "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", | ||
| "properties": { | ||
| "polled": { | ||
| "type": "string", | ||
| "format": "date-time", | ||
| "title": "Polled Timestamp", | ||
| "description": "The time at which the update query which produced this document as executed." | ||
| }, | ||
| "index": { | ||
| "type": "integer", | ||
| "title": "Result Index", | ||
| "description": "The index of this document within the query execution which produced it." | ||
| } | ||
| }, | ||
| "type": "object", | ||
| "required": [ | ||
| "polled", | ||
| "index" | ||
| ] | ||
| }, | ||
| "id": { | ||
| "type": "integer" | ||
| } | ||
| }, | ||
| "x-infer-schema": true | ||
| }, | ||
| "key": [ | ||
| "/id" | ||
| ], | ||
| "projections": null | ||
| }, | ||
| "state_key": "test_capturewithemptypoll_890703" | ||
| } | ||
|
|
25 changes: 25 additions & 0 deletions
25
source-mysql-batch/.snapshots/TestCaptureWithModifications-Capture
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test_capturewithmodifications_786099": 17 Documents | ||
| # ================================ | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 5","id":5,"updated_at":"2025-02-13 12:05:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 6","id":6,"updated_at":"2025-02-13 12:06:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 7","id":7,"updated_at":"2025-02-13 12:07:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 8","id":8,"updated_at":"2025-02-13 12:08:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Initial value for row 9","id":9,"updated_at":"2025-02-13 12:09:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Modified value for row 3","id":3,"updated_at":"2025-02-13 12:15:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Modified value for row 7","id":7,"updated_at":"2025-02-13 12:16:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 10","id":10,"updated_at":"2025-02-13 12:20:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 11","id":11,"updated_at":"2025-02-13 12:21:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 12","id":12,"updated_at":"2025-02-13 12:22:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 13","id":13,"updated_at":"2025-02-13 12:23:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 14","id":14,"updated_at":"2025-02-13 12:24:00"} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test_capturewithmodifications_786099":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:24:00"],"LastPolled":"<TIMESTAMP>"}}} | ||
|
|
58 changes: 58 additions & 0 deletions
58
source-mysql-batch/.snapshots/TestCaptureWithModifications-Discovery
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| Binding 0: | ||
| { | ||
| "resource_config_json": { | ||
| "name": "test_capturewithmodifications_786099", | ||
| "schema": "test", | ||
| "table": "capturewithmodifications_786099", | ||
| "cursor": [ | ||
| "updated_at" | ||
| ] | ||
| }, | ||
| "resource_path": [ | ||
| "test_capturewithmodifications_786099" | ||
| ], | ||
| "collection": { | ||
| "name": "acmeCo/test/test_capturewithmodifications_786099", | ||
| "read_schema_json": { | ||
| "type": "object", | ||
| "required": [ | ||
| "_meta", | ||
| "id" | ||
| ], | ||
| "properties": { | ||
| "_meta": { | ||
| "$schema": "http://json-schema.org/draft/2020-12/schema", | ||
| "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", | ||
| "properties": { | ||
| "polled": { | ||
| "type": "string", | ||
| "format": "date-time", | ||
| "title": "Polled Timestamp", | ||
| "description": "The time at which the update query which produced this document as executed." | ||
| }, | ||
| "index": { | ||
| "type": "integer", | ||
| "title": "Result Index", | ||
| "description": "The index of this document within the query execution which produced it." | ||
| } | ||
| }, | ||
| "type": "object", | ||
| "required": [ | ||
| "polled", | ||
| "index" | ||
| ] | ||
| }, | ||
| "id": { | ||
| "type": "integer" | ||
| } | ||
| }, | ||
| "x-infer-schema": true | ||
| }, | ||
| "key": [ | ||
| "/id" | ||
| ], | ||
| "projections": null | ||
| }, | ||
| "state_key": "test_capturewithmodifications_786099" | ||
| } | ||
|
|
15 changes: 15 additions & 0 deletions
15
source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Capture
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test_capturewithnullcursor_662607": 7 Documents | ||
| # ================================ | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value with NULL cursor","id":0,"sort_col":null} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Another NULL cursor","id":2,"sort_col":null} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Third NULL cursor","id":4,"sort_col":null} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value with cursor 10","id":1,"sort_col":10} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value with cursor 20","id":3,"sort_col":20} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value with cursor 25","id":7,"sort_col":25} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Final value cursor 30","id":9,"sort_col":30} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test_capturewithnullcursor_662607":{"CursorNames":["sort_col"],"CursorValues":[30],"LastPolled":"<TIMESTAMP>"}}} | ||
|
|
58 changes: 58 additions & 0 deletions
58
source-mysql-batch/.snapshots/TestCaptureWithNullCursor-Discovery
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| Binding 0: | ||
| { | ||
| "resource_config_json": { | ||
| "name": "test_capturewithnullcursor_662607", | ||
| "schema": "test", | ||
| "table": "capturewithnullcursor_662607", | ||
| "cursor": [ | ||
| "sort_col" | ||
| ] | ||
| }, | ||
| "resource_path": [ | ||
| "test_capturewithnullcursor_662607" | ||
| ], | ||
| "collection": { | ||
| "name": "acmeCo/test/test_capturewithnullcursor_662607", | ||
| "read_schema_json": { | ||
| "type": "object", | ||
| "required": [ | ||
| "_meta", | ||
| "id" | ||
| ], | ||
| "properties": { | ||
| "_meta": { | ||
| "$schema": "http://json-schema.org/draft/2020-12/schema", | ||
| "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", | ||
| "properties": { | ||
| "polled": { | ||
| "type": "string", | ||
| "format": "date-time", | ||
| "title": "Polled Timestamp", | ||
| "description": "The time at which the update query which produced this document as executed." | ||
| }, | ||
| "index": { | ||
| "type": "integer", | ||
| "title": "Result Index", | ||
| "description": "The index of this document within the query execution which produced it." | ||
| } | ||
| }, | ||
| "type": "object", | ||
| "required": [ | ||
| "polled", | ||
| "index" | ||
| ] | ||
| }, | ||
| "id": { | ||
| "type": "integer" | ||
| } | ||
| }, | ||
| "x-infer-schema": true | ||
| }, | ||
| "key": [ | ||
| "/id" | ||
| ], | ||
| "projections": null | ||
| }, | ||
| "state_key": "test_capturewithnullcursor_662607" | ||
| } | ||
|
|
20 changes: 20 additions & 0 deletions
20
source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Capture
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test_capturewithtwocolumncursor_321285": 12 Documents | ||
| # ================================ | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":1,"col2":1,"data":"Value for row 0","id":0} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":1,"col2":2,"data":"Value for row 1","id":1} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":1,"col2":3,"data":"Value for row 2","id":2} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":2,"col2":1,"data":"Value for row 3","id":3} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":2,"col2":2,"data":"Value for row 4","id":4} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":2,"col2":3,"data":"Value for row 5","id":5} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":3,"col2":1,"data":"Value for row 8","id":8} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":3,"col2":2,"data":"Value for row 9","id":9} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":3,"col2":3,"data":"Value for row 10","id":10} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":4,"col2":1,"data":"Value for row 11","id":11} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":4,"col2":2,"data":"Value for row 12","id":12} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"col1":4,"col2":3,"data":"Value for row 13","id":13} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test_capturewithtwocolumncursor_321285":{"CursorNames":["col1","col2"],"CursorValues":[4,3],"LastPolled":"<TIMESTAMP>"}}} | ||
|
|
59 changes: 59 additions & 0 deletions
59
source-mysql-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| Binding 0: | ||
| { | ||
| "resource_config_json": { | ||
| "name": "test_capturewithtwocolumncursor_321285", | ||
| "schema": "test", | ||
| "table": "capturewithtwocolumncursor_321285", | ||
| "cursor": [ | ||
| "col1", | ||
| "col2" | ||
| ] | ||
| }, | ||
| "resource_path": [ | ||
| "test_capturewithtwocolumncursor_321285" | ||
| ], | ||
| "collection": { | ||
| "name": "acmeCo/test/test_capturewithtwocolumncursor_321285", | ||
| "read_schema_json": { | ||
| "type": "object", | ||
| "required": [ | ||
| "_meta", | ||
| "id" | ||
| ], | ||
| "properties": { | ||
| "_meta": { | ||
| "$schema": "http://json-schema.org/draft/2020-12/schema", | ||
| "$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata", | ||
| "properties": { | ||
| "polled": { | ||
| "type": "string", | ||
| "format": "date-time", | ||
| "title": "Polled Timestamp", | ||
| "description": "The time at which the update query which produced this document as executed." | ||
| }, | ||
| "index": { | ||
| "type": "integer", | ||
| "title": "Result Index", | ||
| "description": "The index of this document within the query execution which produced it." | ||
| } | ||
| }, | ||
| "type": "object", | ||
| "required": [ | ||
| "polled", | ||
| "index" | ||
| ] | ||
| }, | ||
| "id": { | ||
| "type": "integer" | ||
| } | ||
| }, | ||
| "x-infer-schema": true | ||
| }, | ||
| "key": [ | ||
| "/id" | ||
| ], | ||
| "projections": null | ||
| }, | ||
| "state_key": "test_capturewithtwocolumncursor_321285" | ||
| } | ||
|
|
28 changes: 28 additions & 0 deletions
28
source-mysql-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Capture
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test_capturewithupdatedatcursor_792371": 20 Documents | ||
| # ================================ | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 0","id":0,"updated_at":"2025-02-13 12:00:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 1","id":1,"updated_at":"2025-02-13 12:01:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 2","id":2,"updated_at":"2025-02-13 12:02:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 3","id":3,"updated_at":"2025-02-13 12:03:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 4","id":4,"updated_at":"2025-02-13 12:04:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 5","id":5,"updated_at":"2025-02-13 12:05:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 6","id":6,"updated_at":"2025-02-13 12:06:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 7","id":7,"updated_at":"2025-02-13 12:07:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 8","id":8,"updated_at":"2025-02-13 12:08:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 9","id":9,"updated_at":"2025-02-13 12:09:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 10","id":10,"updated_at":"2025-02-13 12:10:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 11","id":11,"updated_at":"2025-02-13 12:11:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 12","id":12,"updated_at":"2025-02-13 12:12:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 13","id":13,"updated_at":"2025-02-13 12:13:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 14","id":14,"updated_at":"2025-02-13 12:14:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 15","id":15,"updated_at":"2025-02-13 12:15:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 16","id":16,"updated_at":"2025-02-13 12:16:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 17","id":17,"updated_at":"2025-02-13 12:17:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 18","id":18,"updated_at":"2025-02-13 12:18:00"} | ||
| {"_meta":{"polled":"<TIMESTAMP>","index":999},"data":"Value for row 19","id":19,"updated_at":"2025-02-13 12:19:00"} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test_capturewithupdatedatcursor_792371":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13 12:19:00"],"LastPolled":"<TIMESTAMP>"}}} | ||
|
|
Oops, something went wrong.