Skip to content

Commit

Permalink
source-bigquery-batch: New keyless capture tests
Browse files Browse the repository at this point in the history
  • Loading branch information
willdonnelly committed Feb 26, 2025
1 parent c381b6a commit 1f1e7be
Show file tree
Hide file tree
Showing 9 changed files with 632 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# ================================
# Collection "acmeCo/test/featureflagkeylessrowid_disabled_906545": 34 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"New row A","value":20}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 8","value":8}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"featureflagkeylessrowid_disabled_906545":{"DocumentCount":9,"LastPolled":"<TIMESTAMP>"}}}

Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
Binding 0:
{
"resource_config_json": {
"name": "featureflagkeylessrowid_disabled_906545",
"schema": "testdata",
"table": "featureflagkeylessrowid_disabled_906545"
},
"resource_path": [
"featureflagkeylessrowid_disabled_906545"
],
"collection": {
"name": "acmeCo/test/featureflagkeylessrowid_disabled_906545",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "featureflagkeylessrowid_disabled_906545"
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# ================================
# Collection "acmeCo/test/featureflagkeylessrowid_enabled_905157": 35 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"c"},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"c"},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"c"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"c"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"c"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"c"},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"c"},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"c"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"c"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"c"},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"u"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"u"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"u"},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"u"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"u"},"data":"New row A","value":20}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"u"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"u"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"d"}}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"featureflagkeylessrowid_enabled_905157":{"DocumentCount":9,"LastPolled":"<TIMESTAMP>"}}}

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
Binding 0:
{
"resource_config_json": {
"name": "featureflagkeylessrowid_enabled_905157",
"schema": "testdata",
"table": "featureflagkeylessrowid_enabled_905157"
},
"resource_path": [
"featureflagkeylessrowid_enabled_905157"
],
"collection": {
"name": "acmeCo/test/featureflagkeylessrowid_enabled_905157",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/row_id"
],
"projections": null
},
"state_key": "featureflagkeylessrowid_enabled_905157"
}

25 changes: 25 additions & 0 deletions source-bigquery-batch/.snapshots/TestKeylessCapture-Capture
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# ================================
# Collection "acmeCo/test/keylesscapture_315710": 17 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","updated_at":"2025-02-13T12:00:00Z","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","updated_at":"2025-02-13T12:01:00Z","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","updated_at":"2025-02-13T12:02:00Z","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","updated_at":"2025-02-13T12:03:00Z","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","updated_at":"2025-02-13T12:04:00Z","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":5},"data":"Additional row 5","updated_at":"2025-02-13T12:10:00Z","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":6},"data":"Additional row 6","updated_at":"2025-02-13T12:11:00Z","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":7},"data":"Additional row 7","updated_at":"2025-02-13T12:12:00Z","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":8},"data":"Additional row 8","updated_at":"2025-02-13T12:13:00Z","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":9},"data":"Additional row 9","updated_at":"2025-02-13T12:14:00Z","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":10},"data":"Additional row 6 (updated)","updated_at":"2025-02-13T12:15:00Z","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":11},"data":"Initial row 4 (updated)","updated_at":"2025-02-13T12:15:00Z","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":12},"data":"Initial row 3 (updated)","updated_at":"2025-02-13T12:15:00Z","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":13},"data":"Additional row 5 (updated)","updated_at":"2025-02-13T12:15:00Z","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":14},"data":"Additional row 7 (updated)","updated_at":"2025-02-13T12:15:00Z","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":15},"data":"Reinserted row 6","updated_at":"2025-02-13T12:20:00Z","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":16},"data":"Reinserted row 4","updated_at":"2025-02-13T12:20:00Z","value":4}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"keylesscapture_315710":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13T12:20:00Z"],"DocumentCount":17,"LastPolled":"<TIMESTAMP>"}}}

72 changes: 72 additions & 0 deletions source-bigquery-batch/.snapshots/TestKeylessCapture-Discovery
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
Binding 0:
{
"resource_config_json": {
"name": "keylesscapture_315710",
"schema": "testdata",
"table": "keylesscapture_315710",
"cursor": [
"updated_at"
]
},
"resource_path": [
"keylesscapture_315710"
],
"collection": {
"name": "acmeCo/test/keylesscapture_315710",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "keylesscapture_315710"
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# ================================
# Collection "acmeCo/test/keylessfullrefreshcapture_941246": 35 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"New row B","value":21}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"New row A","value":20}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"keylessfullrefreshcapture_941246":{"DocumentCount":10,"LastPolled":"<TIMESTAMP>"}}}

Loading

0 comments on commit 1f1e7be

Please sign in to comment.