source-oracle-flashback: float types with precision < 16 are not string

estuary · Dec 16, 2024 · fc8aceb · fc8aceb
1 parent d649c97
commit fc8aceb
Show file tree

Hide file tree

Showing 9 changed files with 64 additions and 31 deletions.
diff --git a/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml
@@ -11,9 +11,7 @@ $defs:
         title: Op
         type: string
       source:
-        anyOf:
-          - $ref: "#/$defs/Source"
-          - type: "null"
+        $ref: "#/$defs/Source"
     required:
       - op
       - source
@@ -46,9 +44,6 @@ properties:
   _meta:
     allOf:
       - $ref: "#/$defs/Meta"
-    default:
-      op: u
-      source: ~
     description: Document metadata
   ID:
     anyOf:
@@ -57,5 +52,7 @@ properties:
     default: ~
     format: number
     title: Id
+required:
+  - _meta
 title: TEST
 type: object
diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml
@@ -11,9 +11,7 @@ $defs:
         title: Op
         type: string
       source:
-        anyOf:
-          - $ref: "#/$defs/Source"
-          - type: "null"
+        $ref: "#/$defs/Source"
     required:
       - op
       - source
@@ -46,9 +44,6 @@ properties:
   _meta:
     allOf:
       - $ref: "#/$defs/Meta"
-    default:
-      op: u
-      source: ~
     description: Document metadata
   ID:
     description: Primary Key
@@ -75,6 +70,19 @@ properties:
     default: ~
     format: number
     title: Float 126
+  FLOAT_16:
+    anyOf:
+      - type: number
+      - type: "null"
+    default: ~
+    title: Float 16
+  FLOAT_63:
+    anyOf:
+      - type: string
+      - type: "null"
+    default: ~
+    format: number
+    title: Float 63
   INTEG:
     anyOf:
       - type: string
@@ -199,6 +207,7 @@ properties:
     default: ~
     title: Vchar2
 required:
+  - _meta
   - ID
 title: TEST_ALL_TYPES
 type: object
diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml
@@ -11,9 +11,7 @@ $defs:
         title: Op
         type: string
       source:
-        anyOf:
-          - $ref: "#/$defs/Source"
-          - type: "null"
+        $ref: "#/$defs/Source"
     required:
       - op
       - source
@@ -46,9 +44,6 @@ properties:
   _meta:
     allOf:
       - $ref: "#/$defs/Meta"
-    default:
-      op: u
-      source: ~
     description: Document metadata
   ID:
     description: Primary Key
@@ -62,6 +57,7 @@ properties:
     default: ~
     title: Str
 required:
+  - _meta
   - ID
 title: TEST_CHANGES
 type: object
diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml
@@ -11,9 +11,7 @@ $defs:
         title: Op
         type: string
       source:
-        anyOf:
-          - $ref: "#/$defs/Source"
-          - type: "null"
+        $ref: "#/$defs/Source"
     required:
       - op
       - source
@@ -46,9 +44,6 @@ properties:
   _meta:
     allOf:
       - $ref: "#/$defs/Meta"
-    default:
-      op: u
-      source: ~
     description: Document metadata
   ID:
     anyOf:
@@ -57,5 +52,7 @@ properties:
     default: ~
     format: number
     title: Id
+required:
+  - _meta
 title: TEST_EMPTY
 type: object
diff --git a/source-oracle-flashback/source_oracle_flashback/models.py b/source-oracle-flashback/source_oracle_flashback/models.py
@@ -291,18 +291,19 @@ def build_table(
         field_type: type[int | str | datetime | float | Decimal]
         field_schema_extra: dict | None = None
 
-        if col.data_type == col.Type.NUMBER and col.data_scale == 0:
-            # datA_precision: 0 defaults to precision 38
+        if col.data_type == col.Type.NUMBER and col.data_scale == 0 :
+            # data_precision: 0 defaults to precision 38
             if col.data_precision > 18 or col.data_precision == 0:
                 field_type = Decimal
                 field_schema_extra = {"format": "number"}
             else:
                 field_type = int
-
         elif col.data_type in (col.Type.DOUBLE, col.Type.NUMBER, col.Type.FLOAT):
-            # Floats cannot be used as keys, so use {type: string, format: number}.
-            field_type = Decimal
-            field_schema_extra = {"format": "number"}
+            if col.data_precision > 18 or col.data_precision == 0:
+                field_type = Decimal
+                field_schema_extra = {"format": "number"}
+            else:
+                field_type = float
         elif col.data_type in (col.Type.CHAR, col.Type.VARCHAR, col.Type.VARCHAR2, col.Type.NCHAR, col.Type.NVARCHAR2):
             field_type = str
         elif col.is_datetime and col.has_timezone:

diff --git a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql
@@ -11,6 +11,8 @@ CREATE TABLE flow_test.test_all_types(
   integ INTEGER,
 	double_precision DOUBLE PRECISION,
 	float_126 FLOAT(126),
+	float_63 FLOAT(63),
+	float_16 FLOAT(16),
 	real_num REAL,
 	datetime DATE,
 	ts TIMESTAMP,

diff --git a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql
@@ -11,6 +11,8 @@ INSERT INTO flow_test.test_all_types(
   integ,
 	double_precision,
 	float_126,
+	float_63,
+	float_16,
 	real_num,
 	datetime,
 	ts,
@@ -32,7 +34,9 @@ INSERT INTO flow_test.test_all_types(
 	1234567891234567891,
 	123456789123456,
 	123456789,
-  18446744073709551615,
+	18446744073709551615,
+	123456789.123456789,
+	123456789.123456789,
 	123456789.123456789,
 	123456789.123456789,
 	123456789.123456789,

diff --git a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json
@@ -5,6 +5,8 @@
       "DATETIME": "2022-01-01T00:00:00",
       "DOUBLE_PRECISION": "123456789.123456789",
       "FLOAT_126": "123456789.123456789",
+      "FLOAT_16": 123460000.0,
+      "FLOAT_63": "123456789.123456789",
       "ID": "1",
       "INTEG": "18446744073709551615",
       "INTERVAL_DAY": "+01 02:03:04.567000",

diff --git a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json
@@ -119,6 +119,31 @@
           "format": "number",
           "title": "Float 126"
         },
+        "FLOAT_16": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Float 16"
+        },
+        "FLOAT_63": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "format": "number",
+          "title": "Float 63"
+        },
         "INTEG": {
           "anyOf": [
             {