From e83b2090e5f732b977904abf1761646eddee8be3 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Fri, 14 Feb 2025 01:54:51 -0600 Subject: [PATCH 1/4] source-postgres-batch: Test suite overhaul This more or less completely replaces the source-postgres-batch test suite, which previously...well, it existed. The new tests should exercise much more of the connector's behavior, including how it discovers and serializes all notable column types. --- .../.snapshots/TestArrayTypes-Capture | 18 + ...ure-Discovery => TestArrayTypes-Discovery} | 10 +- ...pture-Capture => TestAsyncCapture-Capture} | 4 +- ...s-Discovery => TestAsyncCapture-Discovery} | 10 +- .../.snapshots/TestBasicDatatypes-Capture | 9 - .../.snapshots/TestBinaryTypes-Capture | 17 + .../.snapshots/TestBinaryTypes-Discovery | 58 + .../TestCaptureWithEmptyPoll-Capture | 18 + .../TestCaptureWithEmptyPoll-Discovery | 58 + .../.snapshots/TestDateAndTimeTypes-Capture | 18 + .../.snapshots/TestDateAndTimeTypes-Discovery | 58 + source-postgres-batch/.snapshots/TestDecimals | 10 - .../.snapshots/TestFloatNaNs-Capture | 11 - .../.snapshots/TestGeometricTypes-Capture | 15 + .../.snapshots/TestGeometricTypes-Discovery | 58 + .../.snapshots/TestIntegerTypes-Capture | 13 + .../.snapshots/TestIntegerTypes-Discovery | 58 + .../.snapshots/TestJSONColumn | 10 - .../.snapshots/TestJSONTypes-Capture | 18 + ...NaNs-Discovery => TestJSONTypes-Discovery} | 10 +- .../.snapshots/TestKeyDiscovery | 10 +- .../TestModificationsAndDeletions-Capture | 22 + .../TestModificationsAndDeletions-Discovery | 58 + .../.snapshots/TestNetworkTypes-Capture | 18 + .../.snapshots/TestNetworkTypes-Discovery | 58 + .../.snapshots/TestNumericTypes-Capture | 17 + .../.snapshots/TestNumericTypes-Discovery | 58 + .../.snapshots/TestSchemaFilter-FilteredIn | 10 +- .../.snapshots/TestSchemaFilter-Unfiltered | 10 +- .../.snapshots/TestSimpleCapture-Capture | 28 + .../.snapshots/TestSimpleCapture-Discovery | 58 + .../.snapshots/TestStringTypes-Capture | 18 + .../.snapshots/TestStringTypes-Discovery | 58 + .../.snapshots/TestUUIDType-Capture | 19 + .../.snapshots/TestUUIDType-Discovery | 58 + .../.snapshots/TestXMLType-Capture | 19 + .../.snapshots/TestXMLType-Discovery | 58 + source-postgres-batch/driver.go | 10 + source-postgres-batch/main_test.go | 1154 ++++++++++++++--- 39 files changed, 1961 insertions(+), 261 deletions(-) create mode 100644 source-postgres-batch/.snapshots/TestArrayTypes-Capture rename source-postgres-batch/.snapshots/{TestBasicCapture-Discovery => TestArrayTypes-Discovery} (86%) rename source-postgres-batch/.snapshots/{TestBasicCapture-Capture => TestAsyncCapture-Capture} (99%) rename source-postgres-batch/.snapshots/{TestBasicDatatypes-Discovery => TestAsyncCapture-Discovery} (85%) delete mode 100644 source-postgres-batch/.snapshots/TestBasicDatatypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestBinaryTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestBinaryTypes-Discovery create mode 100644 source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Capture create mode 100644 source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery create mode 100644 source-postgres-batch/.snapshots/TestDateAndTimeTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestDateAndTimeTypes-Discovery delete mode 100644 source-postgres-batch/.snapshots/TestDecimals delete mode 100644 source-postgres-batch/.snapshots/TestFloatNaNs-Capture create mode 100644 source-postgres-batch/.snapshots/TestGeometricTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestGeometricTypes-Discovery create mode 100644 source-postgres-batch/.snapshots/TestIntegerTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestIntegerTypes-Discovery delete mode 100644 source-postgres-batch/.snapshots/TestJSONColumn create mode 100644 source-postgres-batch/.snapshots/TestJSONTypes-Capture rename source-postgres-batch/.snapshots/{TestFloatNaNs-Discovery => TestJSONTypes-Discovery} (87%) create mode 100644 source-postgres-batch/.snapshots/TestModificationsAndDeletions-Capture create mode 100644 source-postgres-batch/.snapshots/TestModificationsAndDeletions-Discovery create mode 100644 source-postgres-batch/.snapshots/TestNetworkTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestNetworkTypes-Discovery create mode 100644 source-postgres-batch/.snapshots/TestNumericTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestNumericTypes-Discovery create mode 100644 source-postgres-batch/.snapshots/TestSimpleCapture-Capture create mode 100644 source-postgres-batch/.snapshots/TestSimpleCapture-Discovery create mode 100644 source-postgres-batch/.snapshots/TestStringTypes-Capture create mode 100644 source-postgres-batch/.snapshots/TestStringTypes-Discovery create mode 100644 source-postgres-batch/.snapshots/TestUUIDType-Capture create mode 100644 source-postgres-batch/.snapshots/TestUUIDType-Discovery create mode 100644 source-postgres-batch/.snapshots/TestXMLType-Capture create mode 100644 source-postgres-batch/.snapshots/TestXMLType-Discovery diff --git a/source-postgres-batch/.snapshots/TestArrayTypes-Capture b/source-postgres-batch/.snapshots/TestArrayTypes-Capture new file mode 100644 index 0000000000..9e22801156 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestArrayTypes-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_arraytypes_371544": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"id":0,"int_array":"{1,2,3}","real_array":"{1.1,2.2,3.3}","text_array":"{one,two,three}","txid":999999,"uuid_array":"{550e8400-e29b-41d4-a716-446655440000,6ba7b810-9dad-11d1-80b4-00c04fd430c8}"} +{"_meta":{"polled":"","index":999},"id":1,"int_array":"{}","real_array":"{}","text_array":"{}","txid":999999,"uuid_array":"{}"} +{"_meta":{"polled":"","index":999},"id":2,"int_array":"{1,NULL,3}","real_array":"{1.1,NULL,3.3}","text_array":"{one,NULL,three}","txid":999999,"uuid_array":"{550e8400-e29b-41d4-a716-446655440000,NULL}"} +{"_meta":{"polled":"","index":999},"id":3,"int_array":"{{1,2},{3,4}}","real_array":"{{1.1,2.2},{3.3,4.4}}","text_array":"{{a,b},{c,d}}","txid":999999,"uuid_array":"{{550e8400-e29b-41d4-a716-446655440000},{6ba7b810-9dad-11d1-80b4-00c04fd430c8}}"} +{"_meta":{"polled":"","index":999},"id":4,"int_array":"{2147483647,-2147483648}","real_array":"{Infinity,-Infinity,NaN}","text_array":"{linenbreak,tabthere,\"quote\\\"mark\"}","txid":999999,"uuid_array":"{00000000-0000-0000-0000-000000000000,ffffffff-ffff-ffff-ffff-ffffffffffff}"} +{"_meta":{"polled":"","index":999},"id":5,"int_array":"{42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42}","real_array":"{3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14,3.14}","text_array":"{text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text,text}","txid":999999,"uuid_array":"{550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000,550e8400-e29b-41d4-a716-446655440000}"} +{"_meta":{"polled":"","index":999},"id":6,"int_array":"{1234,5678}","real_array":"{1.234,5.678}","text_array":"{\"Hello, World!\",\"Contains \\\"quotes\\\"\",\"Special chars: @#$%\"}","txid":999999,"uuid_array":"{550e8400-e29b-41d4-a716-446655440000}"} +{"_meta":{"polled":"","index":999},"id":7,"int_array":"{789,12}","real_array":"{7.89,0.12}","text_array":"{\"Hello 世界\",\"Привет мир\",🌍🌎🌏}","txid":999999,"uuid_array":"{6ba7b810-9dad-11d1-80b4-00c04fd430c8}"} +{"_meta":{"polled":"","index":999},"id":8,"int_array":null,"real_array":null,"text_array":null,"txid":999999,"uuid_array":null} +{"_meta":{"polled":"","index":999},"id":9,"int_array":"{42}","real_array":"{3.14159}","text_array":"{\"single element\"}","txid":999999,"uuid_array":"{550e8400-e29b-41d4-a716-446655440000}"} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_arraytypes_371544":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestBasicCapture-Discovery b/source-postgres-batch/.snapshots/TestArrayTypes-Discovery similarity index 86% rename from source-postgres-batch/.snapshots/TestBasicCapture-Discovery rename to source-postgres-batch/.snapshots/TestArrayTypes-Discovery index ea964081dd..ff193607e3 100644 --- a/source-postgres-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-postgres-batch/.snapshots/TestArrayTypes-Discovery @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_basic_capture_826935", + "name": "test_arraytypes_371544", "schema": "test", - "table": "basic_capture_826935", + "table": "arraytypes_371544", "cursor": [ "txid" ] }, "resource_path": [ - "test_basic_capture_826935" + "test_arraytypes_371544" ], "collection": { - "name": "acmeCo/test/test_basic_capture_826935", + "name": "acmeCo/test/test_arraytypes_371544", "read_schema_json": { "type": "object", "required": [ @@ -53,6 +53,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_basic_capture_826935" + "state_key": "test_arraytypes_371544" } diff --git a/source-postgres-batch/.snapshots/TestBasicCapture-Capture b/source-postgres-batch/.snapshots/TestAsyncCapture-Capture similarity index 99% rename from source-postgres-batch/.snapshots/TestBasicCapture-Capture rename to source-postgres-batch/.snapshots/TestAsyncCapture-Capture index be4d318083..3a163d17db 100644 --- a/source-postgres-batch/.snapshots/TestBasicCapture-Capture +++ b/source-postgres-batch/.snapshots/TestAsyncCapture-Capture @@ -1,5 +1,5 @@ # ================================ -# Collection "acmeCo/test/test_basic_capture_826935": 250 Documents +# Collection "acmeCo/test/test_asynccapture_817595": 250 Documents # ================================ {"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0,"txid":999999} {"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1,"txid":999999} @@ -254,5 +254,5 @@ # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test_basic_capture_826935":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} +{"bindingStateV1":{"test_asynccapture_817595":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} diff --git a/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery b/source-postgres-batch/.snapshots/TestAsyncCapture-Discovery similarity index 85% rename from source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery rename to source-postgres-batch/.snapshots/TestAsyncCapture-Discovery index f8fe801706..24ae302cbe 100644 --- a/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-postgres-batch/.snapshots/TestAsyncCapture-Discovery @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_basic_datatypes_13111208", + "name": "test_asynccapture_817595", "schema": "test", - "table": "basic_datatypes_13111208", + "table": "asynccapture_817595", "cursor": [ "txid" ] }, "resource_path": [ - "test_basic_datatypes_13111208" + "test_asynccapture_817595" ], "collection": { - "name": "acmeCo/test/test_basic_datatypes_13111208", + "name": "acmeCo/test/test_asynccapture_817595", "read_schema_json": { "type": "object", "required": [ @@ -53,6 +53,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_basic_datatypes_13111208" + "state_key": "test_asynccapture_817595" } diff --git a/source-postgres-batch/.snapshots/TestBasicDatatypes-Capture b/source-postgres-batch/.snapshots/TestBasicDatatypes-Capture deleted file mode 100644 index a322ab03a2..0000000000 --- a/source-postgres-batch/.snapshots/TestBasicDatatypes-Capture +++ /dev/null @@ -1,9 +0,0 @@ -# ================================ -# Collection "acmeCo/test/test_basic_datatypes_13111208": 1 Documents -# ================================ -{"_meta":{"polled":"","index":999},"a_bool":true,"a_date":"2024-02-26T00:00:00Z","a_real":-12.34000015258789,"a_ts":"2024-02-26T06:34:56-06:00","a_tstz":"2024-02-26T06:34:56-06:00","id":100,"txid":999999} -# ================================ -# Final State Checkpoint -# ================================ -{"bindingStateV1":{"test_basic_datatypes_13111208":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} - diff --git a/source-postgres-batch/.snapshots/TestBinaryTypes-Capture b/source-postgres-batch/.snapshots/TestBinaryTypes-Capture new file mode 100644 index 0000000000..fb978ecb9f --- /dev/null +++ b/source-postgres-batch/.snapshots/TestBinaryTypes-Capture @@ -0,0 +1,17 @@ +# ================================ +# Collection "acmeCo/test/test_binarytypes_537491": 9 Documents +# ================================ +{"_meta":{"polled":"","index":999},"bit3_col":"101","bit_col":"1","bitvar5_col":"10101","bitvar_col":"1010","bool_col":true,"bytea_col":"SGVsbG8=","id":0,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"000","bit_col":"0","bitvar5_col":"00000","bitvar_col":"0","bool_col":false,"bytea_col":"AAAA","id":1,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"111","bit_col":"1","bitvar5_col":"11111","bitvar_col":"111111","bool_col":true,"bytea_col":"////","id":2,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"110","bit_col":"0","bitvar5_col":"10110","bitvar_col":"1100","bool_col":true,"bytea_col":"qrvM","id":3,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"000","bit_col":"0","bitvar5_col":"","bitvar_col":"","bool_col":false,"bytea_col":"","id":4,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":null,"bit_col":null,"bitvar5_col":null,"bitvar_col":null,"bool_col":null,"bytea_col":null,"id":5,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"101","bit_col":"1","bitvar5_col":"10101","bitvar_col":"1010","bool_col":true,"bytea_col":"AAECAwQF/w==","id":6,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"010","bit_col":"0","bitvar5_col":"01010","bitvar_col":"0101","bool_col":false,"bytea_col":"SGVsbG8g5LiW55WM","id":7,"txid":999999} +{"_meta":{"polled":"","index":999},"bit3_col":"111","bit_col":"1","bitvar5_col":"11111","bitvar_col":"1111","bool_col":true,"bytea_col":"AP9/gAEjRWeJq83v","id":8,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_binarytypes_537491":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestBinaryTypes-Discovery b/source-postgres-batch/.snapshots/TestBinaryTypes-Discovery new file mode 100644 index 0000000000..6b9a33b5f0 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestBinaryTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_binarytypes_537491", + "schema": "test", + "table": "binarytypes_537491", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_binarytypes_537491" + ], + "collection": { + "name": "acmeCo/test/test_binarytypes_537491", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_binarytypes_537491" + } + diff --git a/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Capture b/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Capture new file mode 100644 index 0000000000..7f041d6120 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_capturewithemptypoll_890703": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_capturewithemptypoll_890703":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery b/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery new file mode 100644 index 0000000000..85e010a4ff --- /dev/null +++ b/source-postgres-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_capturewithemptypoll_890703", + "schema": "test", + "table": "capturewithemptypoll_890703", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_capturewithemptypoll_890703" + ], + "collection": { + "name": "acmeCo/test/test_capturewithemptypoll_890703", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_capturewithemptypoll_890703" + } + diff --git a/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Capture b/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Capture new file mode 100644 index 0000000000..cadc1b4025 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_dateandtimetypes_307398": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"date_col":"2025-02-14T00:00:00Z","id":0,"interval_col":"1 year 2 mons 3 days 04:05:06","time_col":"14:44:29","timetz_col":"14:44:29-05","ts_col":"2025-02-14T14:44:29Z","tstz_col":"2025-02-13T23:44:29-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"1969-07-20T00:00:00Z","id":1,"interval_col":"50 years","time_col":"20:17:00","timetz_col":"20:17:00-04","ts_col":"1969-07-20T20:17:00Z","tstz_col":"1969-07-20T15:17:00-05:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"2077-07-20T00:00:00Z","id":2,"interval_col":"100 years","time_col":"15:30:00","timetz_col":"15:30:00+09","ts_col":"2077-07-20T15:30:00Z","tstz_col":"2077-07-20T01:30:00-05:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"0001-01-01T00:00:00Z","id":3,"interval_col":"00:00:00.000001","time_col":"00:00:00","timetz_col":"00:00:00+00","ts_col":"0001-01-01T00:00:00Z","tstz_col":"0000-12-31T18:09:24-05:50","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"9999-12-31T00:00:00Z","id":4,"interval_col":"1000 years","time_col":"23:59:59.999999","timetz_col":"23:59:59.999999+00","ts_col":"9999-12-31T23:59:59.999999Z","tstz_col":"9999-12-31T17:59:59.999999-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"2024-02-14T00:00:00Z","id":5,"interval_col":"-178000000 years","time_col":"15:30:45","timetz_col":"15:30:45+00","ts_col":"2024-02-14T15:30:45Z","tstz_col":"2024-02-14T09:30:45-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"2024-02-14T00:00:00Z","id":6,"interval_col":"178000000 years","time_col":"15:30:45","timetz_col":"15:30:45+00","ts_col":"2024-02-14T15:30:45Z","tstz_col":"2024-02-14T09:30:45-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"2024-02-14T00:00:00Z","id":7,"interval_col":"00:00:01.123456","time_col":"15:30:45.123456","timetz_col":"15:30:45.123456+00","ts_col":"2024-02-14T15:30:45.123456Z","tstz_col":"2024-02-14T09:30:45.123456-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":"2024-02-14T00:00:00Z","id":8,"interval_col":"2 years 3 mons 4 days 12:30:45.123456","time_col":"15:30:45","timetz_col":"15:30:45+00","ts_col":"2024-02-14T15:30:45Z","tstz_col":"2024-02-14T09:30:45-06:00","txid":999999} +{"_meta":{"polled":"","index":999},"date_col":null,"id":9,"interval_col":null,"time_col":null,"timetz_col":null,"ts_col":null,"tstz_col":null,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_dateandtimetypes_307398":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Discovery b/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Discovery new file mode 100644 index 0000000000..75e5679d57 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestDateAndTimeTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_dateandtimetypes_307398", + "schema": "test", + "table": "dateandtimetypes_307398", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_dateandtimetypes_307398" + ], + "collection": { + "name": "acmeCo/test/test_dateandtimetypes_307398", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_dateandtimetypes_307398" + } + diff --git a/source-postgres-batch/.snapshots/TestDecimals b/source-postgres-batch/.snapshots/TestDecimals deleted file mode 100644 index c3b6f0866f..0000000000 --- a/source-postgres-batch/.snapshots/TestDecimals +++ /dev/null @@ -1,10 +0,0 @@ -# ================================ -# Collection "acmeCo/test/test_decimals_75319739": 2 Documents -# ================================ -{"_meta":{"polled":"","index":999},"id":0,"txid":999999,"x":"12.3456","y":"1234567890.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"} -{"_meta":{"polled":"","index":999},"id":1,"txid":999999,"x":"98.7654","y":"9876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210.0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"} -# ================================ -# Final State Checkpoint -# ================================ -{"bindingStateV1":{"test_decimals_75319739":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} - diff --git a/source-postgres-batch/.snapshots/TestFloatNaNs-Capture b/source-postgres-batch/.snapshots/TestFloatNaNs-Capture deleted file mode 100644 index 778a498147..0000000000 --- a/source-postgres-batch/.snapshots/TestFloatNaNs-Capture +++ /dev/null @@ -1,11 +0,0 @@ -# ================================ -# Collection "acmeCo/test/test_float_nans_10511": 3 Documents -# ================================ -{"_meta":{"polled":"","index":999},"a_double":"NaN","a_real":2,"id":0,"txid":999999} -{"_meta":{"polled":"","index":999},"a_double":3,"a_real":"NaN","id":1,"txid":999999} -{"_meta":{"polled":"","index":999},"a_double":"-Infinity","a_real":"Infinity","id":2,"txid":999999} -# ================================ -# Final State Checkpoint -# ================================ -{"bindingStateV1":{"test_float_nans_10511":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} - diff --git a/source-postgres-batch/.snapshots/TestGeometricTypes-Capture b/source-postgres-batch/.snapshots/TestGeometricTypes-Capture new file mode 100644 index 0000000000..8f50e4d4c4 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestGeometricTypes-Capture @@ -0,0 +1,15 @@ +# ================================ +# Collection "acmeCo/test/test_geometrictypes_696406": 7 Documents +# ================================ +{"_meta":{"polled":"","index":999},"box_col":"(2,2),(1,1)","circle_col":"\u003c(0,0),1\u003e","id":0,"line_col":"{1,-1,0}","lseg_col":"[(1,1),(2,2)]","path_col":"((1,1),(2,2),(3,3))","point_col":"(1,2)","polygon_col":"((1,1),(2,2),(3,3))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":"(100,100),(-100,-100)","circle_col":"\u003c(5,5),5\u003e","id":1,"line_col":"{0,1,-1}","lseg_col":"[(0,0),(100,100)]","path_col":"((0,0),(1,1),(2,0),(1,-1),(0,0))","point_col":"(0,0)","polygon_col":"((0,0),(2,0),(2,2),(0,2))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":"(-1,-1),(-2,-2)","circle_col":"\u003c(-5,-5),3\u003e","id":2,"line_col":"{1,1,0}","lseg_col":"[(-1,-1),(-2,-2)]","path_col":"((-1,-1),(-2,-2),(-3,-3))","point_col":"(-1,-1)","polygon_col":"((-1,-1),(-1,1),(1,1),(1,-1))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":"(2.5,2.5),(1.5,1.5)","circle_col":"\u003c(2.5,2.5),1.5\u003e","id":3,"line_col":"{1.5,-2.5,3.5}","lseg_col":"[(1.1,1.1),(2.2,2.2)]","path_col":"((1.1,1.1),(2.2,2.2),(3.3,3.3))","point_col":"(1.5,2.5)","polygon_col":"((0,0),(1.5,1.5),(3,0))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":"(9999,9999),(-9999,-9999)","circle_col":"\u003c(0,0),999999\u003e","id":4,"line_col":"{1000,2000,3000}","lseg_col":"[(9999,9999),(-9999,-9999)]","path_col":"((1000,1000),(2000,2000),(3000,3000))","point_col":"(1000000,-1000000)","polygon_col":"((1000,1000),(2000,2000),(3000,1000))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":"(0,0),(0,0)","circle_col":"\u003c(0,0),0\u003e","id":5,"line_col":"{1,0,0}","lseg_col":"[(0,0),(0,1)]","path_col":"[(0,0)]","point_col":"(0,0)","polygon_col":"((0,0),(0,1),(1,0))","txid":999999} +{"_meta":{"polled":"","index":999},"box_col":null,"circle_col":null,"id":6,"line_col":null,"lseg_col":null,"path_col":null,"point_col":null,"polygon_col":null,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_geometrictypes_696406":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestGeometricTypes-Discovery b/source-postgres-batch/.snapshots/TestGeometricTypes-Discovery new file mode 100644 index 0000000000..802aacb62a --- /dev/null +++ b/source-postgres-batch/.snapshots/TestGeometricTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_geometrictypes_696406", + "schema": "test", + "table": "geometrictypes_696406", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_geometrictypes_696406" + ], + "collection": { + "name": "acmeCo/test/test_geometrictypes_696406", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_geometrictypes_696406" + } + diff --git a/source-postgres-batch/.snapshots/TestIntegerTypes-Capture b/source-postgres-batch/.snapshots/TestIntegerTypes-Capture new file mode 100644 index 0000000000..6fa4c9e7d5 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestIntegerTypes-Capture @@ -0,0 +1,13 @@ +# ================================ +# Collection "acmeCo/test/test_integertypes_795898": 5 Documents +# ================================ +{"_meta":{"polled":"","index":999},"bigint_col":42,"bigserial_col":42,"id":0,"int_col":42,"oid_col":42,"serial_col":42,"smallint_col":42,"smallserial_col":42,"txid":999999} +{"_meta":{"polled":"","index":999},"bigint_col":-9223372036854775808,"bigserial_col":1,"id":1,"int_col":-2147483648,"oid_col":0,"serial_col":1,"smallint_col":-32768,"smallserial_col":1,"txid":999999} +{"_meta":{"polled":"","index":999},"bigint_col":9223372036854775807,"bigserial_col":9223372036854775807,"id":2,"int_col":2147483647,"oid_col":4294967295,"serial_col":2147483647,"smallint_col":32767,"smallserial_col":32767,"txid":999999} +{"_meta":{"polled":"","index":999},"bigint_col":0,"bigserial_col":0,"id":3,"int_col":0,"oid_col":0,"serial_col":0,"smallint_col":0,"smallserial_col":0,"txid":999999} +{"_meta":{"polled":"","index":999},"bigint_col":null,"bigserial_col":5,"id":4,"int_col":null,"oid_col":null,"serial_col":5,"smallint_col":null,"smallserial_col":5,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_integertypes_795898":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestIntegerTypes-Discovery b/source-postgres-batch/.snapshots/TestIntegerTypes-Discovery new file mode 100644 index 0000000000..83239819e4 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestIntegerTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_integertypes_795898", + "schema": "test", + "table": "integertypes_795898", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_integertypes_795898" + ], + "collection": { + "name": "acmeCo/test/test_integertypes_795898", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_integertypes_795898" + } + diff --git a/source-postgres-batch/.snapshots/TestJSONColumn b/source-postgres-batch/.snapshots/TestJSONColumn deleted file mode 100644 index 744f6ca4c3..0000000000 --- a/source-postgres-batch/.snapshots/TestJSONColumn +++ /dev/null @@ -1,10 +0,0 @@ -# ================================ -# Collection "acmeCo/test/test_json_column_26214": 2 Documents -# ================================ -{"_meta":{"polled":"","index":999},"data":{"a":123},"id":0,"txid":999999} -{"_meta":{"polled":"","index":999},"data":{"b":456},"id":1,"txid":999999} -# ================================ -# Final State Checkpoint -# ================================ -{"bindingStateV1":{"test_json_column_26214":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} - diff --git a/source-postgres-batch/.snapshots/TestJSONTypes-Capture b/source-postgres-batch/.snapshots/TestJSONTypes-Capture new file mode 100644 index 0000000000..3bfa859e2f --- /dev/null +++ b/source-postgres-batch/.snapshots/TestJSONTypes-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_jsontypes_447235": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"id":0,"json_col":{"number":42,"string":"hello"},"jsonb_col":{"number":42,"string":"hello"},"jsonpath_col":"$.\"number\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":1,"json_col":{"object":{"nested":true},"array":[1,2,3],"null":null,"number":123.456,"string":"test","boolean":false},"jsonb_col":{"null":null,"array":[1,2,3],"number":123.456,"object":{"nested":true},"string":"test","boolean":false},"jsonpath_col":"$.\"object\".\"nested\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":2,"json_col":[1,"two",true,null,{"key":"value"}],"jsonb_col":[1,"two",true,null,{"key":"value"}],"jsonpath_col":"$[4].\"key\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":3,"json_col":{"special\nchars":"tab\there\nand\"quotes\""},"jsonb_col":{"special\nchars":"tab\there\nand\"quotes\""},"jsonpath_col":"$.\"special\\nchars\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":4,"json_col":{"key":"long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string "},"jsonb_col":{"key":"long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string long string "},"jsonpath_col":"$.\"key\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":5,"json_col":{"unicode":"Hello 世界 🌍"},"jsonb_col":{"unicode":"Hello 世界 🌍"},"jsonpath_col":"$.\"unicode\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":6,"json_col":{"a":[{"b":1},{"b":2},{"b":3}]},"jsonb_col":{"a":[{"b":1},{"b":2},{"b":3}]},"jsonpath_col":"$.\"a\"[*].\"b\"?(@ \u003e 1)","txid":999999} +{"_meta":{"polled":"","index":999},"id":7,"json_col":{"empty_obj":{},"empty_arr":[]},"jsonb_col":{"empty_arr":[],"empty_obj":{}},"jsonpath_col":"$.\"empty_arr\"[*]","txid":999999} +{"_meta":{"polled":"","index":999},"id":8,"json_col":{"int_max":2147483647,"int_min":-2147483648,"big_num":1.23456789e+300,"small_num":1.23456789e-300},"jsonb_col":{"big_num":1234567890000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,"int_max":2147483647,"int_min":-2147483648,"small_num":0.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000123456789},"jsonpath_col":"$.\"big_num\"","txid":999999} +{"_meta":{"polled":"","index":999},"id":9,"json_col":null,"jsonb_col":null,"jsonpath_col":null,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_jsontypes_447235":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestFloatNaNs-Discovery b/source-postgres-batch/.snapshots/TestJSONTypes-Discovery similarity index 87% rename from source-postgres-batch/.snapshots/TestFloatNaNs-Discovery rename to source-postgres-batch/.snapshots/TestJSONTypes-Discovery index b41e8f6ff1..ebb9ff6633 100644 --- a/source-postgres-batch/.snapshots/TestFloatNaNs-Discovery +++ b/source-postgres-batch/.snapshots/TestJSONTypes-Discovery @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_float_nans_10511", + "name": "test_jsontypes_447235", "schema": "test", - "table": "float_nans_10511", + "table": "jsontypes_447235", "cursor": [ "txid" ] }, "resource_path": [ - "test_float_nans_10511" + "test_jsontypes_447235" ], "collection": { - "name": "acmeCo/test/test_float_nans_10511", + "name": "acmeCo/test/test_jsontypes_447235", "read_schema_json": { "type": "object", "required": [ @@ -53,6 +53,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_float_nans_10511" + "state_key": "test_jsontypes_447235" } diff --git a/source-postgres-batch/.snapshots/TestKeyDiscovery b/source-postgres-batch/.snapshots/TestKeyDiscovery index 7101f685ec..af2564c619 100644 --- a/source-postgres-batch/.snapshots/TestKeyDiscovery +++ b/source-postgres-batch/.snapshots/TestKeyDiscovery @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_key_discovery_329932", + "name": "test_keydiscovery_608585", "schema": "test", - "table": "key_discovery_329932", + "table": "keydiscovery_608585", "cursor": [ "txid" ] }, "resource_path": [ - "test_key_discovery_329932" + "test_keydiscovery_608585" ], "collection": { - "name": "acmeCo/test/test_key_discovery_329932", + "name": "acmeCo/test/test_keydiscovery_608585", "read_schema_json": { "type": "object", "required": [ @@ -73,6 +73,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_key_discovery_329932" + "state_key": "test_keydiscovery_608585" } diff --git a/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Capture b/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Capture new file mode 100644 index 0000000000..6a5b6db293 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Capture @@ -0,0 +1,22 @@ +# ================================ +# Collection "acmeCo/test/test_modificationsanddeletions_172811": 14 Documents +# ================================ +{"_meta":{"polled":"","index":999},"description":"Initial version of row 0","id":0,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 1","id":1,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 2","id":2,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 3","id":3,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 4","id":4,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 1 (updated)","id":1,"txid":999999,"version":2} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 3 (updated)","id":3,"txid":999999,"version":2} +{"_meta":{"polled":"","index":999},"description":"Reused ID 2","id":2,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Reused ID 4","id":4,"txid":999999,"version":1} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 0 (final update)","id":0,"txid":999999,"version":2} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 1 (updated) (final update)","id":1,"txid":999999,"version":3} +{"_meta":{"polled":"","index":999},"description":"Initial version of row 3 (updated) (final update)","id":3,"txid":999999,"version":3} +{"_meta":{"polled":"","index":999},"description":"Reused ID 2 (final update)","id":2,"txid":999999,"version":2} +{"_meta":{"polled":"","index":999},"description":"Reused ID 4 (final update)","id":4,"txid":999999,"version":2} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_modificationsanddeletions_172811":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Discovery b/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Discovery new file mode 100644 index 0000000000..138cac4225 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestModificationsAndDeletions-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_modificationsanddeletions_172811", + "schema": "test", + "table": "modificationsanddeletions_172811", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_modificationsanddeletions_172811" + ], + "collection": { + "name": "acmeCo/test/test_modificationsanddeletions_172811", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_modificationsanddeletions_172811" + } + diff --git a/source-postgres-batch/.snapshots/TestNetworkTypes-Capture b/source-postgres-batch/.snapshots/TestNetworkTypes-Capture new file mode 100644 index 0000000000..a10c66f765 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestNetworkTypes-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_networktypes_297765": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"cidr_col":"192.168.1.0/24","id":0,"inet_col":"192.168.1.1","macaddr8_col":"08:00:2b:01:02:03:04:05","macaddr_col":"08:00:2b:01:02:03","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"127.0.0.1/32","id":1,"inet_col":"127.0.0.1","macaddr8_col":"00:00:00:00:00:00:00:00","macaddr_col":"00:00:00:00:00:00","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"0.0.0.0/0","id":2,"inet_col":"0.0.0.0","macaddr8_col":"ff:ff:ff:ff:ff:ff:ff:ff","macaddr_col":"ff:ff:ff:ff:ff:ff","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"255.255.255.0/24","id":3,"inet_col":"255.255.255.255","macaddr8_col":"02:42:ac:11:00:02:00:00","macaddr_col":"02:42:ac:11:00:02","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"::/128","id":4,"inet_col":"::1","macaddr8_col":"08:00:2b:01:02:03:04:05","macaddr_col":"08:00:2b:01:02:03","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"2001:db8::/32","id":5,"inet_col":"2001:db8::1","macaddr8_col":"00:11:22:33:44:55:66:77","macaddr_col":"00:11:22:33:44:55","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"fe80::/10","id":6,"inet_col":"fe80::1","macaddr8_col":"52:54:00:ff:fe:01:00:00","macaddr_col":"52:54:00:ff:fe:01","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"10.0.0.0/8","id":7,"inet_col":"192.168.1.5/24","macaddr8_col":"aa:bb:cc:dd:ee:ff:00:11","macaddr_col":"aa:bb:cc:dd:ee:ff","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":"172.16.0.0/12","id":8,"inet_col":"2001:db8::1/64","macaddr8_col":"f0:0d:ca:fe:be:ef:00:00","macaddr_col":"f0:0d:ca:fe:be:ef","txid":999999} +{"_meta":{"polled":"","index":999},"cidr_col":null,"id":9,"inet_col":null,"macaddr8_col":null,"macaddr_col":null,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_networktypes_297765":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestNetworkTypes-Discovery b/source-postgres-batch/.snapshots/TestNetworkTypes-Discovery new file mode 100644 index 0000000000..60751e2adb --- /dev/null +++ b/source-postgres-batch/.snapshots/TestNetworkTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_networktypes_297765", + "schema": "test", + "table": "networktypes_297765", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_networktypes_297765" + ], + "collection": { + "name": "acmeCo/test/test_networktypes_297765", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_networktypes_297765" + } + diff --git a/source-postgres-batch/.snapshots/TestNumericTypes-Capture b/source-postgres-batch/.snapshots/TestNumericTypes-Capture new file mode 100644 index 0000000000..c2e53f3a01 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestNumericTypes-Capture @@ -0,0 +1,17 @@ +# ================================ +# Collection "acmeCo/test/test_numerictypes_559424": 9 Documents +# ================================ +{"_meta":{"polled":"","index":999},"decimal_col":"123.45","double_col":123.456,"id":0,"money_col":"$123.45","numeric_col":"123.45","numeric_large_col":"123.4500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":123.45600128173828,"txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"-999999.99","double_col":-1.7e+308,"id":1,"money_col":"-$92,233,720,368,547,758.08","numeric_col":"-999999.99","numeric_large_col":"-9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999000.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":-3.3999999521443642e+38,"txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"999999.99","double_col":1.7e+308,"id":2,"money_col":"$92,233,720,368,547,758.07","numeric_col":"999999.99","numeric_large_col":"9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999000.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":3.3999999521443642e+38,"txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"0.00","double_col":"NaN","id":3,"money_col":"$0.00","numeric_col":"0.00","numeric_large_col":"0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":"NaN","txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"1.00","double_col":"Infinity","id":4,"money_col":"$1.00","numeric_col":"1.00","numeric_large_col":"1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":"Infinity","txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"-1.00","double_col":"-Infinity","id":5,"money_col":"-$1.00","numeric_col":"-1.00","numeric_large_col":"-1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":"-Infinity","txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"0.00","double_col":0,"id":6,"money_col":"$0.00","numeric_col":"0.00","numeric_large_col":"0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","real_col":0,"txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":null,"double_col":null,"id":7,"money_col":null,"numeric_col":null,"numeric_large_col":null,"real_col":null,"txid":999999} +{"_meta":{"polled":"","index":999},"decimal_col":"0.00","double_col":0,"id":8,"money_col":"$0.00","numeric_col":"0.00","numeric_large_col":"9876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210.0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789","real_col":0,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_numerictypes_559424":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestNumericTypes-Discovery b/source-postgres-batch/.snapshots/TestNumericTypes-Discovery new file mode 100644 index 0000000000..c6813a66d9 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestNumericTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_numerictypes_559424", + "schema": "test", + "table": "numerictypes_559424", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_numerictypes_559424" + ], + "collection": { + "name": "acmeCo/test/test_numerictypes_559424", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_numerictypes_559424" + } + diff --git a/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn index 5b8242806d..809621e0ef 100644 --- a/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_schema_filtering_22492", + "name": "test_schemafilter_428403", "schema": "test", - "table": "schema_filtering_22492", + "table": "schemafilter_428403", "cursor": [ "txid" ] }, "resource_path": [ - "test_schema_filtering_22492" + "test_schemafilter_428403" ], "collection": { - "name": "acmeCo/test/test_schema_filtering_22492", + "name": "acmeCo/test/test_schemafilter_428403", "read_schema_json": { "type": "object", "required": [ @@ -53,6 +53,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_schema_filtering_22492" + "state_key": "test_schemafilter_428403" } diff --git a/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered index 5b8242806d..809621e0ef 100644 --- a/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -1,18 +1,18 @@ Binding 0: { "resource_config_json": { - "name": "test_schema_filtering_22492", + "name": "test_schemafilter_428403", "schema": "test", - "table": "schema_filtering_22492", + "table": "schemafilter_428403", "cursor": [ "txid" ] }, "resource_path": [ - "test_schema_filtering_22492" + "test_schemafilter_428403" ], "collection": { - "name": "acmeCo/test/test_schema_filtering_22492", + "name": "acmeCo/test/test_schemafilter_428403", "read_schema_json": { "type": "object", "required": [ @@ -53,6 +53,6 @@ Binding 0: ], "projections": null }, - "state_key": "test_schema_filtering_22492" + "state_key": "test_schemafilter_428403" } diff --git a/source-postgres-batch/.snapshots/TestSimpleCapture-Capture b/source-postgres-batch/.snapshots/TestSimpleCapture-Capture new file mode 100644 index 0000000000..fbf394a89f --- /dev/null +++ b/source-postgres-batch/.snapshots/TestSimpleCapture-Capture @@ -0,0 +1,28 @@ +# ================================ +# Collection "acmeCo/test/test_simplecapture_140272": 20 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 11","id":11,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 13","id":13,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 14","id":14,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 15","id":15,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 16","id":16,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 17","id":17,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 18","id":18,"txid":999999} +{"_meta":{"polled":"","index":999},"data":"Value for row 19","id":19,"txid":999999} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_simplecapture_140272":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestSimpleCapture-Discovery b/source-postgres-batch/.snapshots/TestSimpleCapture-Discovery new file mode 100644 index 0000000000..3f1b2b8b0d --- /dev/null +++ b/source-postgres-batch/.snapshots/TestSimpleCapture-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_simplecapture_140272", + "schema": "test", + "table": "simplecapture_140272", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_simplecapture_140272" + ], + "collection": { + "name": "acmeCo/test/test_simplecapture_140272", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_simplecapture_140272" + } + diff --git a/source-postgres-batch/.snapshots/TestStringTypes-Capture b/source-postgres-batch/.snapshots/TestStringTypes-Capture new file mode 100644 index 0000000000..6ab465e846 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestStringTypes-Capture @@ -0,0 +1,18 @@ +# ================================ +# Collection "acmeCo/test/test_stringtypes_339419": 10 Documents +# ================================ +{"_meta":{"polled":"","index":999},"bpchar_col":"Hello","char_col":"Hello ","id":0,"text_col":"Hello","txid":999999,"varchar_col":"Hello"} +{"_meta":{"polled":"","index":999},"bpchar_col":"C","char_col":"B ","id":1,"text_col":"D","txid":999999,"varchar_col":"A"} +{"_meta":{"polled":"","index":999},"bpchar_col":"1234567890123456","char_col":"1234567890123456","id":2,"text_col":"1234567890123456","txid":999999,"varchar_col":"1234567890123456"} +{"_meta":{"polled":"","index":999},"bpchar_col":"Hello 世界","char_col":"Hello 世界 ","id":3,"text_col":"Hello 世界","txid":999999,"varchar_col":"Hello 世界"} +{"_meta":{"polled":"","index":999},"bpchar_col":"Tab\tNewline\n","char_col":"Tab\tNewline\n ","id":4,"text_col":"Tab\tNewline\n","txid":999999,"varchar_col":"Tab\tNewline\n"} +{"_meta":{"polled":"","index":999},"bpchar_col":"","char_col":" ","id":5,"text_col":"","txid":999999,"varchar_col":""} +{"_meta":{"polled":"","index":999},"bpchar_col":null,"char_col":null,"id":6,"text_col":null,"txid":999999,"varchar_col":null} +{"_meta":{"polled":"","index":999},"bpchar_col":" trim test ","char_col":" trim test ","id":7,"text_col":" trim test ","txid":999999,"varchar_col":" trim test "} +{"_meta":{"polled":"","index":999},"bpchar_col":"Short","char_col":"Short ","id":8,"text_col":"Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text Long text ","txid":999999,"varchar_col":"Short"} +{"_meta":{"polled":"","index":999},"bpchar_col":"Special \"quotes\"","char_col":"Special \"quotes\"","id":9,"text_col":"Special \"quotes\" and \\backslashes\\","txid":999999,"varchar_col":"Special \"quotes\""} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_stringtypes_339419":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestStringTypes-Discovery b/source-postgres-batch/.snapshots/TestStringTypes-Discovery new file mode 100644 index 0000000000..6e84f0aede --- /dev/null +++ b/source-postgres-batch/.snapshots/TestStringTypes-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_stringtypes_339419", + "schema": "test", + "table": "stringtypes_339419", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_stringtypes_339419" + ], + "collection": { + "name": "acmeCo/test/test_stringtypes_339419", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_stringtypes_339419" + } + diff --git a/source-postgres-batch/.snapshots/TestUUIDType-Capture b/source-postgres-batch/.snapshots/TestUUIDType-Capture new file mode 100644 index 0000000000..e58fd252da --- /dev/null +++ b/source-postgres-batch/.snapshots/TestUUIDType-Capture @@ -0,0 +1,19 @@ +# ================================ +# Collection "acmeCo/test/test_uuidtype_694251": 11 Documents +# ================================ +{"_meta":{"polled":"","index":999},"id":0,"txid":999999,"uuid_col":"d36af820-6ac3-4052-ad19-7eba916a020b"} +{"_meta":{"polled":"","index":999},"id":1,"txid":999999,"uuid_col":"11111111-1111-1111-1111-111111111111"} +{"_meta":{"polled":"","index":999},"id":2,"txid":999999,"uuid_col":"00000000-0000-0000-0000-000000000000"} +{"_meta":{"polled":"","index":999},"id":3,"txid":999999,"uuid_col":"ffffffff-ffff-ffff-ffff-ffffffffffff"} +{"_meta":{"polled":"","index":999},"id":4,"txid":999999,"uuid_col":"a0eebc99-9c0b-11eb-a8b3-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":5,"txid":999999,"uuid_col":"a0eebc99-9c0b-21eb-a8b3-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":6,"txid":999999,"uuid_col":"a0eebc99-9c0b-31eb-a8b3-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":7,"txid":999999,"uuid_col":"a0eebc99-9c0b-41eb-a8b3-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":8,"txid":999999,"uuid_col":"a0eebc99-9c0b-51eb-a8b3-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":9,"txid":999999,"uuid_col":"018df60a-0040-7000-a000-0242ac130003"} +{"_meta":{"polled":"","index":999},"id":10,"txid":999999,"uuid_col":null} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_uuidtype_694251":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestUUIDType-Discovery b/source-postgres-batch/.snapshots/TestUUIDType-Discovery new file mode 100644 index 0000000000..1ee386ecff --- /dev/null +++ b/source-postgres-batch/.snapshots/TestUUIDType-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_uuidtype_694251", + "schema": "test", + "table": "uuidtype_694251", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_uuidtype_694251" + ], + "collection": { + "name": "acmeCo/test/test_uuidtype_694251", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_uuidtype_694251" + } + diff --git a/source-postgres-batch/.snapshots/TestXMLType-Capture b/source-postgres-batch/.snapshots/TestXMLType-Capture new file mode 100644 index 0000000000..6fa54910fa --- /dev/null +++ b/source-postgres-batch/.snapshots/TestXMLType-Capture @@ -0,0 +1,19 @@ +# ================================ +# Collection "acmeCo/test/test_xmltype_831740": 11 Documents +# ================================ +{"_meta":{"polled":"","index":999},"id":0,"txid":999999,"xml_col":"\u003croot\u003e\u003celement\u003eHello\u003c/element\u003e\u003c/root\u003e"} +{"_meta":{"polled":"","index":999},"id":1,"txid":999999,"xml_col":"\u003clibrary\u003e\n \u003cbook id=\"1\"\u003e\n \u003ctitle\u003eSample Book\u003c/title\u003e\n \u003cauthor\u003eJohn Doe\u003c/author\u003e\n \u003cpublished\u003e2024\u003c/published\u003e\n \u003cgenres\u003e\n \u003cgenre\u003eFiction\u003c/genre\u003e\n \u003cgenre\u003eAdventure\u003c/genre\u003e\n \u003c/genres\u003e\n \u003c/book\u003e\n \u003c/library\u003e"} +{"_meta":{"polled":"","index":999},"id":2,"txid":999999,"xml_col":"\u003cuser id=\"123\" active=\"true\"\u003e\n \u003cname\u003eAlice\u003c/name\u003e\n \u003cemail\u003ealice@example.com\u003c/email\u003e\n \u003c/user\u003e"} +{"_meta":{"polled":"","index":999},"id":3,"txid":999999,"xml_col":"\u003ctext\u003e\u003c![CDATA[Special \u0026 \u003ccharacters\u003e \" ' in CDATA]]\u003e\u003c/text\u003e"} +{"_meta":{"polled":"","index":999},"id":4,"txid":999999,"xml_col":"\u003csoap:Envelope xmlns:soap=\"http://www.w3.org/2003/05/soap-envelope\"\u003e\n \u003csoap:Header\u003e\u003cauth\u003etoken123\u003c/auth\u003e\u003c/soap:Header\u003e\n \u003csoap:Body\u003e\u003cmessage\u003eHello World\u003c/message\u003e\u003c/soap:Body\u003e\n \u003c/soap:Envelope\u003e"} +{"_meta":{"polled":"","index":999},"id":5,"txid":999999,"xml_col":"\u003cgreeting language=\"multi\"\u003eHello 世界 🌍\u003c/greeting\u003e"} +{"_meta":{"polled":"","index":999},"id":6,"txid":999999,"xml_col":"\u003cempty/\u003e"} +{"_meta":{"polled":"","index":999},"id":7,"txid":999999,"xml_col":"\u003cdocument\u003e\u003cheader/\u003e\u003cbody/\u003e\u003cfooter/\u003e\u003c/document\u003e"} +{"_meta":{"polled":"","index":999},"id":8,"txid":999999,"xml_col":"\u003croot\u003e\u003ccontent\u003eThis is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. This is a long text content. \u003c/content\u003e\u003c/root\u003e"} +{"_meta":{"polled":"","index":999},"id":9,"txid":999999,"xml_col":"\u003carticle\u003e\n \u003ctitle\u003eMixed Content\u003c/title\u003e\n \u003cpara\u003eThis is \u003cemphasis\u003eimportant\u003c/emphasis\u003e text with \u003clink href=\"#ref\"\u003elinks\u003c/link\u003e.\u003c/para\u003e\n \u003c/article\u003e"} +{"_meta":{"polled":"","index":999},"id":10,"txid":999999,"xml_col":null} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test_xmltype_831740":{"CursorNames":["txid"],"CursorValues":[999999],"LastPolled":""}}} + diff --git a/source-postgres-batch/.snapshots/TestXMLType-Discovery b/source-postgres-batch/.snapshots/TestXMLType-Discovery new file mode 100644 index 0000000000..ab0b920d87 --- /dev/null +++ b/source-postgres-batch/.snapshots/TestXMLType-Discovery @@ -0,0 +1,58 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_xmltype_831740", + "schema": "test", + "table": "xmltype_831740", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_xmltype_831740" + ], + "collection": { + "name": "acmeCo/test/test_xmltype_831740", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "id" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "id": { + "type": "integer" + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ], + "projections": null + }, + "state_key": "test_xmltype_831740" + } + diff --git a/source-postgres-batch/driver.go b/source-postgres-batch/driver.go index 2f841142b2..6ed2bb3ed0 100644 --- a/source-postgres-batch/driver.go +++ b/source-postgres-batch/driver.go @@ -46,6 +46,13 @@ const ( pollingWatchdogTimeout = 5 * time.Minute ) +var ( + // TestShutdownAfterQuery is a test behavior flag which causes the capture to + // shut down after issuing one query to each binding. It is always false in + // normal operation. + TestShutdownAfterQuery = false +) + // BatchSQLDriver represents a generic "batch SQL" capture behavior, parameterized // by a config schema, connect function, and value translation logic. type BatchSQLDriver struct { @@ -597,6 +604,9 @@ func (c *capture) worker(ctx context.Context, binding *bindingInfo) error { if err := c.poll(ctx, binding, queryTemplate); err != nil { return fmt.Errorf("error polling binding %q: %w", res.Name, err) } + if TestShutdownAfterQuery { + return nil // In tests, we want each worker to shut down after one poll + } } return ctx.Err() } diff --git a/source-postgres-batch/main_test.go b/source-postgres-batch/main_test.go index 14c40ac1e7..75a7f39cf8 100644 --- a/source-postgres-batch/main_test.go +++ b/source-postgres-batch/main_test.go @@ -3,11 +3,14 @@ package main import ( "bytes" "context" + "crypto/sha256" "database/sql" + "encoding/binary" "encoding/json" "flag" "fmt" "io" + "math" "os" "regexp" "strings" @@ -42,6 +45,140 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func testCaptureSpec(t testing.TB) *st.CaptureSpec { + t.Helper() + if os.Getenv("TEST_DATABASE") != "yes" { + t.Skipf("skipping %q capture: ${TEST_DATABASE} != \"yes\"", t.Name()) + } + + var endpointSpec = &Config{ + Address: *dbAddress, + User: *dbCaptureUser, + Password: *dbCapturePass, + Database: *dbName, + Advanced: advancedConfig{ + PollSchedule: "200ms", + }, + } + + var sanitizers = make(map[string]*regexp.Regexp) + sanitizers[`"polled":""`] = regexp.MustCompile(`"polled":"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]+:[0-9]+)"`) + sanitizers[`"LastPolled":""`] = regexp.MustCompile(`"LastPolled":"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]+:[0-9]+)"`) + sanitizers[`"index":999`] = regexp.MustCompile(`"index":[0-9]+`) + sanitizers[`"txid":999999`] = regexp.MustCompile(`"txid":[0-9]+`) + sanitizers[`"CursorNames":["txid"],"CursorValues":[999999]`] = regexp.MustCompile(`"CursorNames":\["txid"\],"CursorValues":\[[0-9]+\]`) + + return &st.CaptureSpec{ + Driver: postgresDriver, + EndpointSpec: endpointSpec, + Validator: &st.OrderedCaptureValidator{}, + Sanitizers: sanitizers, + } +} + +func discoverBindings(ctx context.Context, t testing.TB, cs *st.CaptureSpec, matchers ...*regexp.Regexp) []*pf.CaptureSpec_Binding { + t.Helper() + + var discovery = cs.Discover(ctx, t, matchers...) + var bindings []*pf.CaptureSpec_Binding + for _, discovered := range discovery { + var res Resource + require.NoError(t, json.Unmarshal(discovered.ResourceConfigJson, &res)) + bindings = append(bindings, &pf.CaptureSpec_Binding{ + ResourceConfigJson: discovered.ResourceConfigJson, + Collection: pf.CollectionSpec{ + Name: pf.Collection("acmeCo/test/" + discovered.RecommendedName), + ReadSchemaJson: discovered.DocumentSchemaJson, + Key: discovered.Key, + }, + ResourcePath: []string{res.Name}, + StateKey: res.Name, + }) + } + return bindings +} + +func testControlClient(t testing.TB) *sql.DB { + t.Helper() + if os.Getenv("TEST_DATABASE") != "yes" { + t.Skipf("skipping %q capture: ${TEST_DATABASE} != \"yes\"", t.Name()) + } + + var controlUser = *dbControlUser + if controlUser == "" { + controlUser = *dbCaptureUser + } + var controlPass = *dbControlPass + if controlPass == "" { + controlPass = *dbCapturePass + } + var controlURI = fmt.Sprintf(`postgres://%s:%s@%s/%s`, controlUser, controlPass, *dbAddress, *dbName) + log.WithField("uri", controlURI).Debug("opening database control connection") + var conn, err = sql.Open("pgx", controlURI) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + require.NoError(t, conn.Ping()) + return conn +} + +func testTableName(t *testing.T, uniqueID string) (name, id string) { + t.Helper() + const testSchemaName = "test" + var baseName = strings.ToLower(strings.TrimPrefix(t.Name(), "Test")) + for _, str := range []string{"/", "=", "(", ")"} { + baseName = strings.ReplaceAll(baseName, str, "_") + } + return fmt.Sprintf("%s.%s_%s", testSchemaName, baseName, uniqueID), uniqueID +} + +func uniqueTableID(t testing.TB, extra ...string) string { + t.Helper() + var h = sha256.New() + h.Write([]byte(t.Name())) + for _, x := range extra { + h.Write([]byte{':'}) + h.Write([]byte(x)) + } + var x = binary.BigEndian.Uint32(h.Sum(nil)[0:4]) + return fmt.Sprintf("%d", (x%900000)+100000) +} + +func createTestTable(t testing.TB, control *sql.DB, tableName, definition string) { + t.Helper() + executeControlQuery(t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + executeControlQuery(t, control, fmt.Sprintf("CREATE TABLE %s %s", tableName, definition)) + t.Cleanup(func() { executeControlQuery(t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) +} + +func summarizeBindings(t testing.TB, bindings []*pf.CaptureSpec_Binding) string { + var summary = new(strings.Builder) + for idx, binding := range bindings { + fmt.Fprintf(summary, "Binding %d:\n", idx) + bs, err := json.MarshalIndent(binding, " ", " ") + require.NoError(t, err) + io.Copy(summary, bytes.NewReader(bs)) + fmt.Fprintf(summary, "\n") + } + if len(bindings) == 0 { + fmt.Fprintf(summary, "(no bindings)") + } + return summary.String() +} + +func executeControlQuery(t testing.TB, client *sql.DB, query string, args ...interface{}) { + t.Helper() + log.WithFields(log.Fields{"query": query, "args": args}).Debug("executing setup query") + var _, err = client.Exec(query, args...) + require.NoError(t, err) +} + +func setShutdownAfterQuery(t testing.TB, setting bool) { + var oldSetting = TestShutdownAfterQuery + TestShutdownAfterQuery = setting + t.Cleanup(func() { TestShutdownAfterQuery = oldSetting }) +} + +// TestSpec verifies the connector's response to the Spec RPC against a snapshot. func TestSpec(t *testing.T) { response, err := postgresDriver.Spec(context.Background(), &pc.Request_Spec{}) require.NoError(t, err) @@ -51,19 +188,39 @@ func TestSpec(t *testing.T) { cupaloy.SnapshotT(t, string(formatted)) } -func TestBasicCapture(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "826935" - var tableName = fmt.Sprintf("test.basic_capture_%s", uniqueID) +// TestSimpleCapture exercises the simplest use-case of a capture first doing an initial +// backfill and subsequently capturing new rows using the default XMIN cursor. +func TestSimpleCapture(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT)") - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, data TEXT)", tableName)) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for i := 0; i < 10; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s VALUES ($1, $2)", tableName), i, fmt.Sprintf("Value for row %d", i)) + } + cs.Capture(ctx, t, nil) + for i := 10; i < 20; i++ { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s VALUES ($1, $2)", tableName), i, fmt.Sprintf("Value for row %d", i)) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestAsyncCapture performs a capture with periodic restarts, in parallel with a bunch of inserts. +func TestAsyncCapture(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT)") // Discover the table and verify discovery snapshot - cs.Bindings = discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID)) - t.Run("Discovery", func(t *testing.T) { snapshotBindings(t, cs.Bindings) }) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) t.Run("Capture", func(t *testing.T) { // Spawn a worker thread which will insert 50 rows of data in parallel with the capture. @@ -71,7 +228,7 @@ func TestBasicCapture(t *testing.T) { go func() { for i := 0; i < 250; i++ { time.Sleep(100 * time.Millisecond) - executeControlQuery(ctx, t, control, fmt.Sprintf("INSERT INTO %s VALUES ($1, $2)", tableName), i, fmt.Sprintf("Value for row %d", i)) + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s VALUES ($1, $2)", tableName), i, fmt.Sprintf("Value for row %d", i)) log.WithField("i", i).Debug("inserted row") } time.Sleep(1 * time.Second) @@ -88,233 +245,852 @@ func TestBasicCapture(t *testing.T) { }) } -func TestBasicDatatypes(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "13111208" - var tableName = fmt.Sprintf("test.basic_datatypes_%s", uniqueID) +// TestKeyDiscovery exercises the connector's ability to discover types of primary key columns. +func TestKeyDiscovery(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + k_smallint SMALLINT, + k_int INTEGER, + k_bigint BIGINT, + k_bool BOOLEAN, + k_str VARCHAR(8), + data TEXT, + PRIMARY KEY (k_smallint, k_int, k_bigint, k_bool, k_str) + )`) + cupaloy.SnapshotT(t, summarizeBindings(t, discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)))) +} - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, a_real REAL, a_bool BOOL, a_date DATE, a_ts TIMESTAMP, a_tstz TIMESTAMPTZ)", tableName)) +// TestSchemaFilter exercises the 'discover_schemas' advanced option. +func TestSchemaFilter(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, "(id INTEGER PRIMARY KEY, data TEXT)") - // Discover the table and verify discovery snapshot - cs.Bindings = discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID)) - t.Run("Discovery", func(t *testing.T) { snapshotBindings(t, cs.Bindings) }) + // Run discovery with several schema filters and snapshot the results + t.Run("Unfiltered", func(t *testing.T) { + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{} + cupaloy.SnapshotT(t, summarizeBindings(t, discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)))) + }) + t.Run("FilteredOut", func(t *testing.T) { + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"foo", "bar"} + cupaloy.SnapshotT(t, summarizeBindings(t, discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)))) + }) + t.Run("FilteredIn", func(t *testing.T) { + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"foo", "test"} + cupaloy.SnapshotT(t, summarizeBindings(t, discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)))) + }) +} - t.Run("Capture", func(t *testing.T) { - executeControlQuery(ctx, t, control, fmt.Sprintf("INSERT INTO %s(id, a_real, a_bool, a_date, a_ts, a_tstz) VALUES ($1,$2,$3,$4,$5,$6)", tableName), - 100, -12.34, true, "2024-02-26", time.Date(2024, 02, 26, 12, 34, 56, 00, time.UTC), time.Date(2024, 02, 26, 12, 34, 56, 00, time.UTC)) +// TestIntegerTypes exercises discovery and capture of the integer types +// INTEGER, SMALLINT, BIGINT, SERIAL, SMALLSERIAL, BIGSERIAL, and OID. +func TestIntegerTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + int_col INTEGER, + smallint_col SMALLINT, + bigint_col BIGINT, + serial_col SERIAL, + smallserial_col SMALLSERIAL, + bigserial_col BIGSERIAL, + oid_col OID + )`) + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - // Run the capture for 5 seconds, which should be plenty to pull down a few rows. - var captureCtx, cancelCapture = context.WithCancel(ctx) - time.AfterFunc(5*time.Second, cancelCapture) - cs.Capture(captureCtx, t, nil) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + {0, 42, 42, 42, 42, 42, 42, 42}, // Regular row with normal values + {1, -2147483648, -32768, -9223372036854775808, 1, 1, 1, 0}, // Minimum values + {2, 2147483647, 32767, 9223372036854775807, 2147483647, 32767, 9223372036854775807, 4294967295}, // Maximum values + {3, 0, 0, 0, 0, 0, 0, 0}, // Zero values + {4, nil, nil, nil, 5, 5, 5, nil}, // Null values (except for SERIAL columns which can't be null) + } { + executeControlQuery(t, control, fmt.Sprintf("INSERT INTO %s (id, int_col, smallint_col, bigint_col, serial_col, smallserial_col, bigserial_col, oid_col) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", tableName), row...) + } + cs.Capture(ctx, t, nil) cupaloy.SnapshotT(t, cs.Summary()) }) } -func TestDecimals(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "75319739" - var tableName = fmt.Sprintf("test.decimals_%s", uniqueID) +// TestNumericTypes exercises discovery and capture of the numeric types +// REAL, DOUBLE PRECISION, DECIMAL, NUMERIC, NUMERIC(200, 100), and MONEY. +func TestNumericTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + real_col REAL, + double_col DOUBLE PRECISION, + decimal_col DECIMAL(10,2), + numeric_col NUMERIC(10,2), + numeric_large_col NUMERIC(200,100), + money_col MONEY + )`) + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Regular values + {0, 123.456, 123.456, 123.45, 123.45, 123.45, 123.45}, + + // Minimum/maximum values + {1, -3.4e38, -1.7e308, -999999.99, -999999.99, + "-9.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999e+99", + "-92233720368547758.08"}, + {2, 3.4e38, 1.7e308, 999999.99, 999999.99, + "9.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999e+99", + "92233720368547758.07"}, - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, x NUMERIC(6, 4), y NUMERIC(200, 100))", tableName)) + // Special floating point values + {3, float32(math.NaN()), math.NaN(), 0.0, 0.0, 0.0, 0.0}, + {4, float32(math.Inf(1)), math.Inf(1), 1.0, 1.0, 1.0, 1.0}, + {5, float32(math.Inf(-1)), math.Inf(-1), -1.0, -1.0, -1.0, -1.0}, - cs.Bindings = discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID)) + // Zero values + {6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, - var insertQuery = fmt.Sprintf("INSERT INTO %s VALUES ($1,$2,$3)", tableName) - executeControlQuery(ctx, t, control, insertQuery, 0, 12.3456, 1234567890) - executeControlQuery(ctx, t, control, insertQuery, 1, 98.7654, "9876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210.0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") + // Null values + {7, nil, nil, nil, nil, nil, nil}, - // Run the capture for 5 seconds, which should be plenty to pull down a few rows. - var captureCtx, cancelCapture = context.WithCancel(ctx) - time.AfterFunc(5*time.Second, cancelCapture) - cs.Capture(captureCtx, t, nil) - cupaloy.SnapshotT(t, cs.Summary()) + // Very precise decimal number + {8, 0.0, 0.0, 0.0, 0.0, + "9876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210.0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", + 0.0}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, real_col, double_col, decimal_col, numeric_col, + numeric_large_col, money_col + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) } -func TestJSONColumn(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "26214" - var tableName = fmt.Sprintf("test.json_column_%s", uniqueID) +// TestBinaryTypes exercises discovery and capture of the binary types +// BOOLEAN, BIT, BIT(3), BIT VARYING, BIT VARYING(5), and BYTEA. +func TestBinaryTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + bool_col BOOLEAN, + bit_col BIT, + bit3_col BIT(3), + bitvar_col BIT VARYING, + bitvar5_col BIT VARYING(5), + bytea_col BYTEA + )`) - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, data JSON)", tableName)) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - cs.Bindings = discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID)) - executeControlQuery(ctx, t, control, fmt.Sprintf(`INSERT INTO %s VALUES (0, '{"a": 123}'), (1, ' {"b": 456}')`, tableName)) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + {0, true, "1", "101", "1010", "10101", []byte("Hello")}, + {1, false, "0", "000", "0", "00000", []byte{0, 0, 0}}, + {2, true, "1", "111", "111111", "11111", []byte{255, 255, 255}}, + {3, true, "0", "110", "1100", "10110", []byte{0xAA, 0xBB, 0xCC}}, + {4, false, "0", "000", "", "", []byte{}}, + {5, nil, nil, nil, nil, nil, nil}, + {6, true, "1", "101", "1010", "10101", []byte{0, 1, 2, 3, 4, 5, 0xFF}}, + {7, false, "0", "010", "0101", "01010", []byte("Hello 世界")}, + {8, true, "1", "111", "1111", "11111", + []byte{0x00, 0xFF, 0x7F, 0x80, 0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, bool_col, bit_col, bit3_col, bitvar_col, + bitvar5_col, bytea_col + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} + +// TestStringTypes exercises discovery and capture of the string types +// VARCHAR(16), CHAR(16), BPCHAR, and TEXT. +func TestStringTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + varchar_col VARCHAR(16), + char_col CHAR(16), + bpchar_col BPCHAR, + text_col TEXT + )`) + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - // Run the capture for 1 second, which should be plenty to pull down a few rows. - var captureCtx, cancelCapture = context.WithCancel(ctx) - time.AfterFunc(1*time.Second, cancelCapture) - cs.Capture(captureCtx, t, nil) - cupaloy.SnapshotT(t, cs.Summary()) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + {0, "Hello", "Hello", "Hello", "Hello"}, + {1, "A", "B", "C", "D"}, + {2, "1234567890123456", "1234567890123456", "1234567890123456", "1234567890123456"}, + {3, "Hello 世界", "Hello 世界", "Hello 世界", "Hello 世界"}, + {4, "Tab\tNewline\n", "Tab\tNewline\n", "Tab\tNewline\n", "Tab\tNewline\n"}, + {5, "", "", "", ""}, + {6, nil, nil, nil, nil}, + {7, " trim test ", " trim test ", " trim test ", " trim test "}, + {8, "Short", "Short", "Short", strings.Repeat("Long text ", 1000)}, + {9, `Special "quotes"`, `Special "quotes"`, `Special "quotes"`, `Special "quotes" and \backslashes\`}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, varchar_col, char_col, bpchar_col, text_col + ) VALUES ($1, $2, $3, $4, $5)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) } -func TestFloatNaNs(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "10511" - var tableName = fmt.Sprintf("test.float_nans_%s", uniqueID) +// TestDateAndTimeTypes exercises discovery and capture of the date and time types +// DATE, TIMESTAMP, TIMESTAMP WITH TIME ZONE, TIME, TIME WITH TIME ZONE, and INTERVAL. +func TestDateAndTimeTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + date_col DATE, + ts_col TIMESTAMP, + tstz_col TIMESTAMP WITH TIME ZONE, + time_col TIME, + timetz_col TIME WITH TIME ZONE, + interval_col INTERVAL + )`) + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, a_real REAL, a_double DOUBLE PRECISION)", tableName)) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + var tokyo, _ = time.LoadLocation("Asia/Tokyo") + for _, row := range [][]any{ + {0, "2025-02-14", "2025-02-14 14:44:29", + time.Date(2025, 2, 14, 14, 44, 29, 0, tokyo), + "14:44:29", "14:44:29-05:00", "1 year 2 months 3 days 4 hours 5 minutes 6 seconds"}, + {1, "1969-07-20", "1969-07-20 20:17:00", + time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC), + "20:17:00", "20:17:00-04:00", "50 years"}, + {2, "2077-07-20", "2077-07-20 15:30:00", + time.Date(2077, 7, 20, 15, 30, 0, 0, tokyo), + "15:30:00", "15:30:00+09:00", "100 years"}, + {3, "0001-01-01", "0001-01-01 00:00:00", + time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + "00:00:00", "00:00:00Z", "1 microsecond"}, + {4, "9999-12-31", "9999-12-31 23:59:59.999999", + time.Date(9999, 12, 31, 23, 59, 59, 999999000, time.UTC), + "23:59:59.999999", "23:59:59.999999Z", "1000 years"}, + {5, "2024-02-14", "2024-02-14 15:30:45", + time.Date(2024, 2, 14, 15, 30, 45, 0, time.UTC), + "15:30:45", "15:30:45Z", "-178000000 years"}, + {6, "2024-02-14", "2024-02-14 15:30:45", + time.Date(2024, 2, 14, 15, 30, 45, 0, time.UTC), + "15:30:45", "15:30:45Z", "178000000 years"}, + {7, "2024-02-14", "2024-02-14 15:30:45.123456", + time.Date(2024, 2, 14, 15, 30, 45, 123456000, time.UTC), + "15:30:45.123456", "15:30:45.123456Z", "1.123456 seconds"}, + {8, "2024-02-14", "2024-02-14 15:30:45", + time.Date(2024, 2, 14, 15, 30, 45, 0, time.UTC), + "15:30:45", "15:30:45Z", + "2 years 3 months 4 days 12 hours 30 minutes 45.123456 seconds"}, + {9, nil, nil, nil, nil, nil, nil}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, date_col, ts_col, tstz_col, time_col, + timetz_col, interval_col + ) VALUES ($1, $2, $3, $4, $5, $6, $7)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) +} - cs.Bindings = discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID)) +// TestGeometricTypes exercises discovery and capture of the geometric types +// POINT, LINE, LSEG, BOX, PATH, POLYGON, and CIRCLE. +func TestGeometricTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + point_col POINT, + line_col LINE, + lseg_col LSEG, + box_col BOX, + path_col PATH, + polygon_col POLYGON, + circle_col CIRCLE + )`) - t.Run("Discovery", func(t *testing.T) { snapshotBindings(t, cs.Bindings) }) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) t.Run("Capture", func(t *testing.T) { - executeControlQuery(ctx, t, control, fmt.Sprintf(`INSERT INTO %s VALUES (0, 2.0, 'NaN'), (1, 'NaN', 3.0), (2, 'Infinity', '-Infinity')`, tableName)) + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Regular geometric values + {0, + "(1,2)", // POINT: (x,y) + "{1,-1,0}", // LINE: {A,B,C} where Ax + By + C = 0 + "[(1,1),(2,2)]", // LSEG: segment from point 1 to point 2 + "((1,1),(2,2))", // BOX: upper right and lower left corners + "((1,1),(2,2),(3,3))", // PATH: series of points forming a path + "((1,1),(2,2),(3,3))", // POLYGON: series of points forming a polygon + "<(0,0),1>", // CIRCLE: center point and radius + }, - // Run the capture for 1 second, which should be plenty to pull down a few rows. - var captureCtx, cancelCapture = context.WithCancel(ctx) - time.AfterFunc(1*time.Second, cancelCapture) - cs.Capture(captureCtx, t, nil) + // Complex shapes + {1, + "(0,0)", + "{0,1,-1}", + "[(0,0),(100,100)]", + "((100,100),(-100,-100))", + "((0,0),(1,1),(2,0),(1,-1),(0,0))", // Closed path + "((0,0),(2,0),(2,2),(0,2))", // Square + "<(5,5),5>", + }, + + // Negative coordinates + {2, + "(-1,-1)", + "{1,1,0}", + "[(-1,-1),(-2,-2)]", + "((-1,-1),(-2,-2))", + "((-1,-1),(-2,-2),(-3,-3))", + "((-1,-1),(-1,1),(1,1),(1,-1))", + "<(-5,-5),3>", + }, + + // Decimal coordinates + {3, + "(1.5,2.5)", + "{1.5,-2.5,3.5}", + "[(1.1,1.1),(2.2,2.2)]", + "((1.5,1.5),(2.5,2.5))", + "((1.1,1.1),(2.2,2.2),(3.3,3.3))", + "((0.0,0.0),(1.5,1.5),(3.0,0.0))", + "<(2.5,2.5),1.5>", + }, + + // Large coordinates + {4, + "(1000000,-1000000)", + "{1000,2000,3000}", + "[(9999,9999),(-9999,-9999)]", + "((9999,9999),(-9999,-9999))", + "((1000,1000),(2000,2000),(3000,3000))", + "((1000,1000),(2000,2000),(3000,1000))", + "<(0,0),999999>", + }, + + // Special cases + {5, + "(0,0)", // Origin point + "{1,0,0}", // Vertical line x=0 + "[(0,0),(0,1)]", // Vertical segment + "((0,0),(0,0))", // Box with zero area + "[(0,0)]", // Single point path (square brackets for open path) + "((0,0),(0,1),(1,0))", // Triangle + "<(0,0),0>", // Circle with zero radius + }, + + // Null values + {6, nil, nil, nil, nil, nil, nil, nil}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, point_col, line_col, lseg_col, box_col, + path_col, polygon_col, circle_col + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) cupaloy.SnapshotT(t, cs.Summary()) }) } -func TestSchemaFilter(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "22492" - var tableName = fmt.Sprintf("test.schema_filtering_%s", uniqueID) +// TestNetworkTypes exercises discovery and capture of the network address types +// INET, CIDR, MACADDR, and MACADDR8. +func TestNetworkTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + inet_col INET, + cidr_col CIDR, + macaddr_col MACADDR, + macaddr8_col MACADDR8 + )`) - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(id INTEGER PRIMARY KEY, data TEXT)", tableName)) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - // Run discovery with several schema filters and snapshot the results - t.Run("Unfiltered", func(t *testing.T) { - cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{} - snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) - }) - t.Run("FilteredOut", func(t *testing.T) { - cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"foo", "bar"} - snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) - }) - t.Run("FilteredIn", func(t *testing.T) { - cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"foo", "test"} - snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Regular values + {0, "192.168.1.1", "192.168.1.0/24", "08:00:2b:01:02:03", "08:00:2b:01:02:03:04:05"}, + + // IPv4 addresses and networks + {1, "127.0.0.1", "127.0.0.1/32", "00:00:00:00:00:00", "00:00:00:00:00:00:00:00"}, + {2, "0.0.0.0", "0.0.0.0/0", "ff:ff:ff:ff:ff:ff", "ff:ff:ff:ff:ff:ff:ff:ff"}, + {3, "255.255.255.255", "255.255.255.0/24", "02:42:ac:11:00:02", "02:42:ac:11:00:02:00:00"}, + + // IPv6 addresses and networks + {4, "::1", "::/128", "08:00:2b:01:02:03", "08:00:2b:01:02:03:04:05"}, + {5, "2001:db8::1", "2001:db8::/32", "00:11:22:33:44:55", "00:11:22:33:44:55:66:77"}, + {6, "fe80::1", "fe80::/10", "52:54:00:ff:fe:01", "52:54:00:ff:fe:01:00:00"}, + + // INET with explicit subnet masks + {7, "192.168.1.5/24", "10.0.0.0/8", "aa:bb:cc:dd:ee:ff", "aa:bb:cc:dd:ee:ff:00:11"}, + {8, "2001:db8::1/64", "172.16.0.0/12", "f0:0d:ca:fe:be:ef", "f0:0d:ca:fe:be:ef:00:00"}, + + // Null values + {9, nil, nil, nil, nil}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, inet_col, cidr_col, macaddr_col, macaddr8_col + ) VALUES ($1, $2, $3, $4, $5)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) }) } -func TestKeyDiscovery(t *testing.T) { - var ctx, cs = context.Background(), testCaptureSpec(t) - var control = testControlClient(ctx, t) - var uniqueID = "329932" - var tableName = fmt.Sprintf("test.key_discovery_%s", uniqueID) +// TestJSONTypes exercises discovery and capture of the JSON types +// JSON, JSONB, and JSONPATH. +func TestJSONTypes(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + json_col JSON, + jsonb_col JSONB, + jsonpath_col JSONPATH + )`) - executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) - t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(k_smallint SMALLINT, k_int INTEGER, k_bigint BIGINT, k_bool BOOLEAN, k_str VARCHAR(8), data TEXT, PRIMARY KEY (k_smallint, k_int, k_bigint, k_bool, k_str))", tableName)) + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"test"} - snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) -} + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Simple values + {0, `{"number": 42, "string": "hello"}`, + `{"number": 42, "string": "hello"}`, + `$.number`}, -func testControlClient(ctx context.Context, t testing.TB) *sql.DB { - t.Helper() - if os.Getenv("TEST_DATABASE") != "yes" { - t.Skipf("skipping %q capture: ${TEST_DATABASE} != \"yes\"", t.Name()) - } + // Complex nested structure + {1, `{ + "object": {"nested": true}, + "array": [1,2,3], + "null": null, + "number": 123.456, + "string": "test", + "boolean": false + }`, `{ + "object": {"nested": true}, + "array": [1,2,3], + "null": null, + "number": 123.456, + "string": "test", + "boolean": false + }`, `$.object.nested`}, - var controlUser = *dbControlUser - if controlUser == "" { - controlUser = *dbCaptureUser - } - var controlPass = *dbControlPass - if controlPass == "" { - controlPass = *dbCapturePass - } - var controlURI = fmt.Sprintf(`postgres://%s:%s@%s/%s`, controlUser, controlPass, *dbAddress, *dbName) - log.WithField("uri", controlURI).Debug("opening database control connection") - var conn, err = sql.Open("pgx", controlURI) - require.NoError(t, err) - t.Cleanup(func() { conn.Close() }) - require.NoError(t, conn.PingContext(ctx)) - return conn + // Array with mixed types + {2, `[1, "two", true, null, {"key": "value"}]`, + `[1, "two", true, null, {"key": "value"}]`, + `$[4].key`}, + + // Special characters + {3, `{"special\nchars": "tab\there\nand\"quotes\""}`, + `{"special\nchars": "tab\there\nand\"quotes\""}`, + `$."special\nchars"`}, + + // Very large structure + {4, fmt.Sprintf(`{"key": "%s"}`, strings.Repeat("long string ", 1000)), + fmt.Sprintf(`{"key": "%s"}`, strings.Repeat("long string ", 1000)), + `$.key`}, + + // Unicode content + {5, `{"unicode": "Hello 世界 🌍"}`, + `{"unicode": "Hello 世界 🌍"}`, + `$.unicode`}, + + // Complex JSONPATH expressions + {6, `{"a": [{"b": 1}, {"b": 2}, {"b": 3}]}`, + `{"a": [{"b": 1}, {"b": 2}, {"b": 3}]}`, + `$.a[*].b ? (@ > 1)`}, + + // Empty objects and arrays + {7, `{"empty_obj": {}, "empty_arr": []}`, + `{"empty_obj": {}, "empty_arr": []}`, + `$.empty_arr[*]`}, + + // Numeric edge cases + {8, `{ + "int_max": 2147483647, + "int_min": -2147483648, + "big_num": 1.23456789e+300, + "small_num": 1.23456789e-300 + }`, `{ + "int_max": 2147483647, + "int_min": -2147483648, + "big_num": 1.23456789e+300, + "small_num": 1.23456789e-300 + }`, `$.big_num`}, + + // Null values + {9, nil, nil, nil}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s ( + id, json_col, jsonb_col, jsonpath_col + ) VALUES ($1, $2, $3, $4)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) } -func executeControlQuery(ctx context.Context, t testing.TB, client *sql.DB, query string, args ...interface{}) { - t.Helper() - log.WithFields(log.Fields{"query": query, "args": args}).Debug("executing setup query") - var _, err = client.ExecContext(ctx, query, args...) - require.NoError(t, err) +// TestUUIDType exercises discovery and capture of the UUID type. +func TestUUIDType(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + uuid_col UUID + )`) + + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) + + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Standard UUID v4 format (random) + {0, "d36af820-6ac3-4052-ad19-7eba916a020b"}, + + // UUID with all characters as same value + {1, "11111111-1111-1111-1111-111111111111"}, + {2, "00000000-0000-0000-0000-000000000000"}, + {3, "ffffffff-ffff-ffff-ffff-ffffffffffff"}, + + // Different versions of UUID + {4, "a0eebc99-9c0b-11eb-a8b3-0242ac130003"}, // v1 time-based + {5, "a0eebc99-9c0b-21eb-a8b3-0242ac130003"}, // v2 DCE security + {6, "a0eebc99-9c0b-31eb-a8b3-0242ac130003"}, // v3 name-based MD5 + {7, "a0eebc99-9c0b-41eb-a8b3-0242ac130003"}, // v4 random + {8, "a0eebc99-9c0b-51eb-a8b3-0242ac130003"}, // v5 name-based SHA-1 + {9, "018df60a-0040-7000-a000-0242ac130003"}, // v7 time-ordered + + // Null value + {10, nil}, + } { + executeControlQuery(t, control, fmt.Sprintf(` + INSERT INTO %s (id, uuid_col) VALUES ($1, $2)`, tableName), + row...) + } + cs.Capture(ctx, t, nil) + cupaloy.SnapshotT(t, cs.Summary()) + }) } -func testCaptureSpec(t testing.TB) *st.CaptureSpec { - t.Helper() - if os.Getenv("TEST_DATABASE") != "yes" { - t.Skipf("skipping %q capture: ${TEST_DATABASE} != \"yes\"", t.Name()) - } +// TestXMLType exercises discovery and capture of the XML type. +func TestXMLType(t *testing.T) { + var ctx, cs, control = context.Background(), testCaptureSpec(t), testControlClient(t) + var tableName, uniqueID = testTableName(t, uniqueTableID(t)) + createTestTable(t, control, tableName, `( + id INTEGER PRIMARY KEY, + xml_col XML + )`) - var endpointSpec = &Config{ - Address: *dbAddress, - User: *dbCaptureUser, - Password: *dbCapturePass, - Database: *dbName, - Advanced: advancedConfig{ - PollSchedule: "200ms", - }, - } + cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) + t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - var sanitizers = make(map[string]*regexp.Regexp) - sanitizers[`"polled":""`] = regexp.MustCompile(`"polled":"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]+:[0-9]+)"`) - sanitizers[`"LastPolled":""`] = regexp.MustCompile(`"LastPolled":"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]+:[0-9]+)"`) - sanitizers[`"index":999`] = regexp.MustCompile(`"index":[0-9]+`) - sanitizers[`"txid":999999`] = regexp.MustCompile(`"txid":[0-9]+`) - sanitizers[`"CursorNames":["txid"],"CursorValues":[999999]`] = regexp.MustCompile(`"CursorNames":\["txid"\],"CursorValues":\[[0-9]+\]`) + t.Run("Capture", func(t *testing.T) { + setShutdownAfterQuery(t, true) + for _, row := range [][]any{ + // Simple XML + {0, "Hello"}, - return &st.CaptureSpec{ - Driver: postgresDriver, - EndpointSpec: endpointSpec, - Validator: &st.OrderedCaptureValidator{}, - Sanitizers: sanitizers, - } + // Complex nested structure + {1, ` + + Sample Book + John Doe + 2024 + + Fiction + Adventure + + + `}, + + // XML with attributes + {2, ` + Alice + alice@example.com + `}, + + // XML with special characters + {3, ` " ' in CDATA]]>`}, + + // XML with namespaces + {4, ` + token123 + Hello World + `}, + + // XML with unicode characters + {5, `Hello 世界 🌍`}, + + // Empty elements + {6, ``}, + + // Self-closing tags + {7, `