From 73281a8808d061aebeed4e11d5044808cd4871cd Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Wed, 26 Feb 2025 17:18:36 -0600 Subject: [PATCH] source-redshift-batch: Require `row_id` only when `keyless_row_id` --- .../.snapshots/TestBasicCapture-Discovery | 3 +-- .../.snapshots/TestBasicDatatypes-Discovery | 3 +-- .../.snapshots/TestFeatureFlagUseSchemaInference-Discovery | 3 +-- source-redshift-batch/.snapshots/TestFloatNaNs-Discovery | 3 +-- source-redshift-batch/.snapshots/TestKeyDiscovery | 3 +-- .../.snapshots/TestKeylessCapture-Discovery | 3 +-- source-redshift-batch/.snapshots/TestKeylessDiscovery | 3 +-- .../.snapshots/TestKeylessFullRefreshCapture-Discovery | 3 +-- .../.snapshots/TestQueryTemplateOverride-Discovery | 3 +-- .../.snapshots/TestSchemaFilter-FilteredIn | 3 +-- .../.snapshots/TestSchemaFilter-Unfiltered | 3 +-- source-redshift-batch/driver.go | 7 +++++-- 12 files changed, 16 insertions(+), 24 deletions(-) diff --git a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery index dfc9b63c4..f1c7ddbc6 100644 --- a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery index e4a194e45..e7c7e9406 100644 --- a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "a_bool": { diff --git a/source-redshift-batch/.snapshots/TestFeatureFlagUseSchemaInference-Discovery b/source-redshift-batch/.snapshots/TestFeatureFlagUseSchemaInference-Discovery index a9b91877f..5821624e5 100644 --- a/source-redshift-batch/.snapshots/TestFeatureFlagUseSchemaInference-Discovery +++ b/source-redshift-batch/.snapshots/TestFeatureFlagUseSchemaInference-Discovery @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestFloatNaNs-Discovery b/source-redshift-batch/.snapshots/TestFloatNaNs-Discovery index b61c6c6b0..1c959369b 100644 --- a/source-redshift-batch/.snapshots/TestFloatNaNs-Discovery +++ b/source-redshift-batch/.snapshots/TestFloatNaNs-Discovery @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "a_double": { diff --git a/source-redshift-batch/.snapshots/TestKeyDiscovery b/source-redshift-batch/.snapshots/TestKeyDiscovery index 1090c63f2..7920ef46d 100644 --- a/source-redshift-batch/.snapshots/TestKeyDiscovery +++ b/source-redshift-batch/.snapshots/TestKeyDiscovery @@ -56,8 +56,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestKeylessCapture-Discovery b/source-redshift-batch/.snapshots/TestKeylessCapture-Discovery index 652ec0c7d..d43d76fd2 100644 --- a/source-redshift-batch/.snapshots/TestKeylessCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestKeylessCapture-Discovery @@ -54,8 +54,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestKeylessDiscovery b/source-redshift-batch/.snapshots/TestKeylessDiscovery index 60d322b1d..bd45d96a4 100644 --- a/source-redshift-batch/.snapshots/TestKeylessDiscovery +++ b/source-redshift-batch/.snapshots/TestKeylessDiscovery @@ -51,8 +51,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "v_bigint": { diff --git a/source-redshift-batch/.snapshots/TestKeylessFullRefreshCapture-Discovery b/source-redshift-batch/.snapshots/TestKeylessFullRefreshCapture-Discovery index c96f749fa..1747838d7 100644 --- a/source-redshift-batch/.snapshots/TestKeylessFullRefreshCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestKeylessFullRefreshCapture-Discovery @@ -51,8 +51,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestQueryTemplateOverride-Discovery b/source-redshift-batch/.snapshots/TestQueryTemplateOverride-Discovery index a9c2f97a3..0ac59e0e3 100644 --- a/source-redshift-batch/.snapshots/TestQueryTemplateOverride-Discovery +++ b/source-redshift-batch/.snapshots/TestQueryTemplateOverride-Discovery @@ -54,8 +54,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn index 08b2a886e..bdeff5d5a 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered index 08b2a886e..bdeff5d5a 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -52,8 +52,7 @@ Binding 0: "type": "object", "required": [ "polled", - "index", - "row_id" + "index" ] }, "data": { diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index 699895d1d..380d031cf 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -122,13 +122,16 @@ var ( fallbackKeyOld = []string{} ) -func generateCollectionSchema(keyColumns []string, columnTypes map[string]columnType, useSchemaInference bool) (json.RawMessage, error) { +func generateCollectionSchema(cfg *Config, keyColumns []string, columnTypes map[string]columnType, useSchemaInference bool) (json.RawMessage, error) { // Generate schema for the metadata via reflection var reflector = jsonschema.Reflector{ ExpandedStruct: true, DoNotReference: true, } var metadataSchema = reflector.ReflectFromType(reflect.TypeOf(documentMetadata{})) + if !cfg.Advanced.parsedFeatureFlags["keyless_row_id"] { // Don't include row_id as required on old captures with keyless_row_id off + metadataSchema.Required = slices.DeleteFunc(metadataSchema.Required, func(s string) bool { return s == "row_id" }) + } metadataSchema.Definitions = nil metadataSchema.AdditionalProperties = nil @@ -285,7 +288,7 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove columnTypes[column.Name] = column.DataType } - generatedSchema, err := generateCollectionSchema(keyColumns, columnTypes, cfg.Advanced.parsedFeatureFlags["use_schema_inference"]) + generatedSchema, err := generateCollectionSchema(&cfg, keyColumns, columnTypes, cfg.Advanced.parsedFeatureFlags["use_schema_inference"]) if err != nil { log.WithFields(log.Fields{"table": tableID, "err": err}).Warn("unable to generate collection schema") continue