From 087807ed592f2c6b0432a690130184d456ede7dc Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Mon, 16 Dec 2024 21:45:25 +0000 Subject: [PATCH] source-oracle-flashback: float types with precision < 18 are not string --- .../acmeCo/FLOW_CAPTURE_TEST.schema.yaml | 9 +++---- .../FLOW_TEST_TEST_ALL_TYPES.schema.yaml | 21 +++++++++++----- .../acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml | 8 ++---- .../acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml | 9 +++---- .../source_oracle_flashback/models.py | 13 +++++----- .../tests/db_seeds/create_test_all_types.sql | 2 ++ .../tests/db_seeds/insert_test_all_types.sql | 6 ++++- .../snapshots__capture_all_types__stdout.json | 2 ++ .../snapshots__discover__stdout.json | 25 +++++++++++++++++++ 9 files changed, 64 insertions(+), 31 deletions(-) diff --git a/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml index a05b213b8b..5da67d3d14 100644 --- a/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: anyOf: @@ -57,5 +52,7 @@ properties: default: ~ format: number title: Id +required: + - _meta title: TEST type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml index d99481103c..68b60cc9b5 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: description: Primary Key @@ -75,6 +70,19 @@ properties: default: ~ format: number title: Float 126 + FLOAT_16: + anyOf: + - type: number + - type: "null" + default: ~ + title: Float 16 + FLOAT_63: + anyOf: + - type: string + - type: "null" + default: ~ + format: number + title: Float 63 INTEG: anyOf: - type: string @@ -199,6 +207,7 @@ properties: default: ~ title: Vchar2 required: + - _meta - ID title: TEST_ALL_TYPES type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml index 697db644a0..ff77516f58 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: description: Primary Key @@ -62,6 +57,7 @@ properties: default: ~ title: Str required: + - _meta - ID title: TEST_CHANGES type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml index e74cf32133..541edaaef5 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: anyOf: @@ -57,5 +52,7 @@ properties: default: ~ format: number title: Id +required: + - _meta title: TEST_EMPTY type: object diff --git a/source-oracle-flashback/source_oracle_flashback/models.py b/source-oracle-flashback/source_oracle_flashback/models.py index 329fcdf822..751cbb129e 100644 --- a/source-oracle-flashback/source_oracle_flashback/models.py +++ b/source-oracle-flashback/source_oracle_flashback/models.py @@ -291,18 +291,19 @@ def build_table( field_type: type[int | str | datetime | float | Decimal] field_schema_extra: dict | None = None - if col.data_type == col.Type.NUMBER and col.data_scale == 0: - # datA_precision: 0 defaults to precision 38 + if col.data_type == col.Type.NUMBER and col.data_scale == 0 : + # data_precision: 0 defaults to precision 38 if col.data_precision > 18 or col.data_precision == 0: field_type = Decimal field_schema_extra = {"format": "number"} else: field_type = int - elif col.data_type in (col.Type.DOUBLE, col.Type.NUMBER, col.Type.FLOAT): - # Floats cannot be used as keys, so use {type: string, format: number}. - field_type = Decimal - field_schema_extra = {"format": "number"} + if col.data_precision > 18 or col.data_precision == 0: + field_type = Decimal + field_schema_extra = {"format": "number"} + else: + field_type = float elif col.data_type in (col.Type.CHAR, col.Type.VARCHAR, col.Type.VARCHAR2, col.Type.NCHAR, col.Type.NVARCHAR2): field_type = str elif col.is_datetime and col.has_timezone: diff --git a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql index 15b284c794..8269ca0056 100644 --- a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql @@ -11,6 +11,8 @@ CREATE TABLE flow_test.test_all_types( integ INTEGER, double_precision DOUBLE PRECISION, float_126 FLOAT(126), + float_63 FLOAT(63), + float_16 FLOAT(16), real_num REAL, datetime DATE, ts TIMESTAMP, diff --git a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql index 9a4f42b254..ee91f4b228 100644 --- a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql @@ -11,6 +11,8 @@ INSERT INTO flow_test.test_all_types( integ, double_precision, float_126, + float_63, + float_16, real_num, datetime, ts, @@ -32,7 +34,9 @@ INSERT INTO flow_test.test_all_types( 1234567891234567891, 123456789123456, 123456789, - 18446744073709551615, + 18446744073709551615, + 123456789.123456789, + 123456789.123456789, 123456789.123456789, 123456789.123456789, 123456789.123456789, diff --git a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json index 72a29cc708..0bbc108809 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json @@ -5,6 +5,8 @@ "DATETIME": "2022-01-01T00:00:00", "DOUBLE_PRECISION": "123456789.123456789", "FLOAT_126": "123456789.123456789", + "FLOAT_16": 123460000.0, + "FLOAT_63": "123456789.123456789", "ID": "1", "INTEG": "18446744073709551615", "INTERVAL_DAY": "+01 02:03:04.567000", diff --git a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json index c945a30d78..17777f10a3 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json @@ -119,6 +119,31 @@ "format": "number", "title": "Float 126" }, + "FLOAT_16": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Float 16" + }, + "FLOAT_63": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "format": "number", + "title": "Float 63" + }, "INTEG": { "anyOf": [ {