diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py index ec7dbc8251b200..8850cb3e183ed4 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py @@ -24,12 +24,19 @@ # For SQL server, the default collation rules mean that all identifiers (schema, table, column names) # are case preserving but case insensitive. "mssql", + # Oracle automatically converts unquoted identifiers to uppercase. + # https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Database-Object-Names-and-Qualifiers.html#GUID-3C59E44A-5140-4BCA-B9E1-3039C8050C49 + # In our Oracle connector, we then normalize column names to lowercase. This behavior + # actually comes from the underlying Oracle sqlalchemy dialect. + # https://github.com/sqlalchemy/sqlalchemy/blob/d9b4d8ff3aae504402d324f3ebf0b8faff78f5dc/lib/sqlalchemy/dialects/oracle/base.py#L2579 + "oracle", } DIALECTS_WITH_DEFAULT_UPPERCASE_COLS = { # In some dialects, column identifiers are effectively case insensitive # because they are automatically converted to uppercase. Most other systems # automatically lowercase unquoted identifiers. "snowflake", + "oracle", } assert DIALECTS_WITH_DEFAULT_UPPERCASE_COLS.issubset( DIALECTS_WITH_CASE_INSENSITIVE_COLS diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index 8cf69535f30f66..ed9f73b76f78fd 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -17,7 +17,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -33,7 +33,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -67,7 +67,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -83,7 +83,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -99,7 +99,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -122,7 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -138,7 +138,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -154,7 +154,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -172,7 +172,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -193,7 +193,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -209,7 +209,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -272,7 +272,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -290,7 +290,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -315,7 +315,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -331,7 +331,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -394,7 +394,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -412,7 +412,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -437,7 +437,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -453,7 +453,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -519,7 +519,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -537,7 +537,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -580,7 +580,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -596,7 +596,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -619,7 +619,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -635,7 +635,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -651,7 +651,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -669,7 +669,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -690,7 +690,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -706,7 +706,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -769,7 +769,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -787,7 +787,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -812,7 +812,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -828,7 +828,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -891,7 +891,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -909,7 +909,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -934,7 +934,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -950,7 +950,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1016,7 +1016,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1034,7 +1034,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1052,7 +1052,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1077,7 +1077,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1107,11 +1107,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),mock_column1)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),mock_column1)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29" @@ -1119,11 +1119,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),mock_column2)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),mock_column2)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29" @@ -1133,7 +1133,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1161,7 +1161,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1177,26 +1177,26 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),mock_column2)" }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),mock_column2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1212,7 +1212,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1242,11 +1242,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),mock_column1)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),mock_column1)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29" @@ -1254,11 +1254,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),mock_column2)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),mock_column2)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29" @@ -1268,7 +1268,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1296,7 +1296,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1312,26 +1312,26 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),mock_column2)" }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),mock_column2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1347,7 +1347,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1363,7 +1363,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } }, @@ -1379,7 +1379,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "runId": "oracle-2022_02_03-07_00_00-77emm3", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index d57ea0e21479da..8ef68c5cb107c7 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -17,7 +17,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -33,7 +33,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -67,7 +67,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -83,7 +83,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -99,7 +99,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -122,7 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -138,7 +138,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -154,7 +154,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -172,7 +172,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -193,7 +193,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -209,7 +209,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -272,7 +272,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -290,7 +290,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -315,7 +315,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -331,7 +331,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -394,7 +394,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -412,7 +412,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -437,7 +437,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -453,7 +453,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -519,7 +519,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -537,7 +537,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -580,7 +580,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -596,7 +596,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -619,7 +619,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -635,7 +635,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -651,7 +651,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -669,7 +669,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -690,7 +690,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -706,7 +706,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -769,7 +769,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -787,7 +787,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -812,7 +812,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -828,7 +828,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -891,7 +891,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -909,7 +909,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -934,7 +934,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -950,7 +950,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1016,7 +1016,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1034,7 +1034,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1052,7 +1052,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1077,7 +1077,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1107,11 +1107,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column1)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),mock_column1)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29" @@ -1119,11 +1119,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column2)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),mock_column2)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29" @@ -1133,7 +1133,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1161,7 +1161,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1177,26 +1177,26 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column2)" }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),mock_column2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1212,7 +1212,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1242,11 +1242,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column1)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),mock_column1)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29" @@ -1254,11 +1254,11 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column2)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),mock_column2)" ], "confidenceScore": 0.2, "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29" @@ -1268,7 +1268,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1296,7 +1296,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1312,26 +1312,26 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),mock_column2)" }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),mock_column1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),mock_column2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1347,7 +1347,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1363,7 +1363,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } }, @@ -1379,7 +1379,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "runId": "oracle-2022_02_03-07_00_00-fl452l", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json index 2f154e9b49d316..b5e3512e526e09 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json @@ -3,58 +3,58 @@ "query_type_props": { "kind": "VIEW" }, - "query_fingerprint": "00400d6c706790ec92f8f1a9fbe41e05573168b071f32f9dac714180c3ef3b86", + "query_fingerprint": "5f1a47abe1e58158645efc579a96305ebe97a3b7658e4c3b39d0d3b39d87655b", "in_tables": [ - "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:mysql,scott.emp,PROD)" ], "out_tables": [ - "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:mysql,vsal,PROD)" ], "column_lineage": [ { "downstream": { - "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,vsal,PROD)", "column": "Department", "column_type": null, "native_column_type": null }, "upstreams": [ { - "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)", - "column": "DEPTNO" + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,scott.emp,PROD)", + "column": "deptno" } ] }, { "downstream": { - "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,vsal,PROD)", "column": "Employees", "column_type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "native_column_type": "DOUBLE PRECISION" + "native_column_type": "DOUBLE" }, "upstreams": [] }, { "downstream": { - "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,vsal,PROD)", "column": "Salary", "column_type": null, "native_column_type": null }, "upstreams": [ { - "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)", - "column": "SAL" + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,scott.emp,PROD)", + "column": "sal" } ] } ], "debug_info": { "confidence": 0.2, - "generalized_statement": "CREATE VIEW vsal AS SELECT a.deptno AS \"Department\", a.num_emp / b.total_count AS \"Employees\", a.sal_sum / b.total_sal AS \"Salary\" FROM (SELECT deptno, COUNT() AS num_emp, SUM(sal) AS sal_sum FROM scott.emp WHERE city = ? GROUP BY deptno) a, (SELECT COUNT() AS total_count, SUM(sal) AS total_sal FROM scott.emp WHERE city = ?) b" + "generalized_statement": "CREATE VIEW vsal AS SELECT a.deptno AS `Department`, a.num_emp / b.total_count AS `Employees`, a.sal_sum / b.total_sal AS `Salary` FROM (SELECT deptno, COUNT() AS num_emp, SUM(sal) AS sal_sum FROM scott.emp WHERE city = ? GROUP BY deptno) AS a, (SELECT COUNT() AS total_count, SUM(sal) AS total_sal FROM scott.emp WHERE city = ?) AS b" } } \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_oracle_case_insensitive_cols.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_oracle_case_insensitive_cols.json new file mode 100644 index 00000000000000..ac6c0d5fd0e7fa --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_oracle_case_insensitive_cols.json @@ -0,0 +1,37 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "c7f48ff25eae801c8bbbe7f86ec725efd4da6ccbdb243e42f6c06209ad2a4cb2", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:oracle,abc.employees,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "employee_type_number", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,abc.employees,PROD)", + "column": "employee_type" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,abc.employees,PROD)", + "column": "job_title" + } + ] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT CASE WHEN (employee_TYPE = ?) THEN (CASE WHEN job_title IN (?) THEN ? ELSE ? END) ELSE (CASE WHEN job_title IN (?) THEN ? ELSE ? END) END AS employee_type_number FROM ABC.employees" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_complex_ctes.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_complex_ctes.json index 0fb0860e2c0702..96457dc891a719 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_complex_ctes.json +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_complex_ctes.json @@ -11,28 +11,28 @@ { "downstream": { "table": null, - "column": "COL1", + "column": "col1", "column_type": null, "native_column_type": null }, "upstreams": [ { "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table1,PROD)", - "column": "COL1" + "column": "col1" } ] }, { "downstream": { "table": null, - "column": "COL3", + "column": "col3", "column_type": null, "native_column_type": null }, "upstreams": [ { "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table2,PROD)", - "column": "COL3" + "column": "col3" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json index 7c2d48a455c031..6dfe2d247faa68 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json @@ -11,28 +11,28 @@ { "downstream": { "table": null, - "column": "COL1", + "column": "col1", "column_type": null, "native_column_type": null }, "upstreams": [ { "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table1,PROD)", - "column": "COL1" + "column": "col1" } ] }, { "downstream": { "table": null, - "column": "COL3", + "column": "col3", "column_type": null, "native_column_type": null }, "upstreams": [ { "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table2,PROD)", - "column": "COL3" + "column": "col3" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 0a78095c7547b3..30f8998ea046d4 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -7,7 +7,7 @@ RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens" -def test_invalid_sql(): +def test_invalid_sql() -> None: assert_sql_result( """ SELECT as ' @@ -19,7 +19,7 @@ def test_invalid_sql(): ) -def test_select_max(): +def test_select_max() -> None: # The COL2 should get normalized to col2. assert_sql_result( """ @@ -31,7 +31,7 @@ def test_select_max(): ) -def test_select_max_with_schema(): +def test_select_max_with_schema() -> None: # Note that `this_will_not_resolve` will be dropped from the result because it's not in the schema. assert_sql_result( """ @@ -49,7 +49,7 @@ def test_select_max_with_schema(): ) -def test_select_count(): +def test_select_count() -> None: assert_sql_result( """ SELECT @@ -63,7 +63,7 @@ def test_select_count(): ) -def test_select_with_ctes(): +def test_select_with_ctes() -> None: assert_sql_result( """ WITH cte1 AS ( @@ -84,7 +84,7 @@ def test_select_with_ctes(): ) -def test_select_with_complex_ctes(): +def test_select_with_complex_ctes() -> None: # This one has group bys in the CTEs, which means they can't be collapsed into the main query. assert_sql_result( """ @@ -108,7 +108,7 @@ def test_select_with_complex_ctes(): ) -def test_multiple_select_subqueries(): +def test_multiple_select_subqueries() -> None: assert_sql_result( """ SELECT SUM((SELECT max(a) a from x) + (SELECT min(b) b from x) + c) AS y FROM x @@ -118,7 +118,7 @@ def test_multiple_select_subqueries(): ) -def test_create_view_as_select(): +def test_create_view_as_select() -> None: assert_sql_result( """ CREATE VIEW vsal @@ -138,12 +138,12 @@ def test_create_view_as_select(): WHERE city = 'NYC') b ; """, - dialect="oracle", + dialect="mysql", expected_file=RESOURCE_DIR / "test_create_view_as_select.json", ) -def test_insert_as_select(): +def test_insert_as_select() -> None: # Note: this also tests lineage with case statements. assert_sql_result( @@ -179,7 +179,7 @@ def test_insert_as_select(): ) -def test_insert_with_column_list(): +def test_insert_with_column_list() -> None: assert_sql_result( """\ insert into downstream (a, c) select a, c from upstream2 @@ -189,7 +189,7 @@ def test_insert_with_column_list(): ) -def test_select_with_full_col_name(): +def test_select_with_full_col_name() -> None: # In this case, `widget` is a struct column. # This also tests the `default_db` functionality. assert_sql_result( @@ -210,7 +210,7 @@ def test_select_with_full_col_name(): ) -def test_select_from_struct_subfields(): +def test_select_from_struct_subfields() -> None: # In this case, `widget` is a column name. assert_sql_result( """ @@ -235,7 +235,7 @@ def test_select_from_struct_subfields(): ) -def test_select_from_union(): +def test_select_from_union() -> None: assert_sql_result( """ SELECT @@ -263,7 +263,7 @@ def test_select_from_union(): ) -def test_select_ambiguous_column_no_schema(): +def test_select_ambiguous_column_no_schema() -> None: assert_sql_result( """ select A, B, C from t1 inner join t2 on t1.id = t2.id @@ -273,7 +273,7 @@ def test_select_ambiguous_column_no_schema(): ) -def test_merge_from_union(): +def test_merge_from_union() -> None: # TODO: We don't support merge statements yet, but the union should still get handled. assert_sql_result( @@ -302,7 +302,7 @@ def test_merge_from_union(): ) -def test_expand_select_star_basic(): +def test_expand_select_star_basic() -> None: assert_sql_result( """ SELECT @@ -329,7 +329,7 @@ def test_expand_select_star_basic(): ) -def test_create_table_ddl(): +def test_create_table_ddl() -> None: assert_sql_result( """ CREATE TABLE IF NOT EXISTS costs ( @@ -344,7 +344,7 @@ def test_create_table_ddl(): ) -def test_snowflake_column_normalization(): +def test_snowflake_column_normalization() -> None: # Technically speaking this is incorrect since the column names are different and both quoted. assert_sql_result( @@ -367,7 +367,7 @@ def test_snowflake_column_normalization(): ) -def test_snowflake_ctas_column_normalization(): +def test_snowflake_ctas_column_normalization() -> None: # For CTAS statements, we also should try to match the output table's # column name casing. This is technically incorrect since we have the # exact column names from the query, but necessary to match our column @@ -401,7 +401,7 @@ def test_snowflake_ctas_column_normalization(): ) -def test_snowflake_case_statement(): +def test_snowflake_case_statement() -> None: assert_sql_result( """ SELECT @@ -430,7 +430,7 @@ def test_snowflake_case_statement(): @pytest.mark.skip(reason="We don't handle the unnest lineage correctly") -def test_bigquery_unnest_columns(): +def test_bigquery_unnest_columns() -> None: assert_sql_result( """ SELECT @@ -468,7 +468,7 @@ def test_bigquery_unnest_columns(): ) -def test_bigquery_create_view_with_cte(): +def test_bigquery_create_view_with_cte() -> None: assert_sql_result( """ CREATE VIEW `my-proj-2`.dataset.my_view AS @@ -506,7 +506,7 @@ def test_bigquery_create_view_with_cte(): ) -def test_bigquery_nested_subqueries(): +def test_bigquery_nested_subqueries() -> None: assert_sql_result( """ SELECT * @@ -529,7 +529,7 @@ def test_bigquery_nested_subqueries(): ) -def test_bigquery_sharded_table_normalization(): +def test_bigquery_sharded_table_normalization() -> None: assert_sql_result( """ SELECT * @@ -546,7 +546,7 @@ def test_bigquery_sharded_table_normalization(): ) -def test_bigquery_from_sharded_table_wildcard(): +def test_bigquery_from_sharded_table_wildcard() -> None: assert_sql_result( """ SELECT * @@ -563,7 +563,7 @@ def test_bigquery_from_sharded_table_wildcard(): ) -def test_bigquery_partitioned_table_insert(): +def test_bigquery_partitioned_table_insert() -> None: assert_sql_result( """ SELECT * @@ -580,7 +580,7 @@ def test_bigquery_partitioned_table_insert(): ) -def test_bigquery_star_with_replace(): +def test_bigquery_star_with_replace() -> None: assert_sql_result( """ CREATE VIEW `my-project.my-dataset.test_table` AS @@ -602,7 +602,7 @@ def test_bigquery_star_with_replace(): ) -def test_bigquery_view_from_union(): +def test_bigquery_view_from_union() -> None: assert_sql_result( """ CREATE VIEW my_view as @@ -625,7 +625,7 @@ def test_bigquery_view_from_union(): ) -def test_snowflake_default_normalization(): +def test_snowflake_default_normalization() -> None: assert_sql_result( """ create table active_customer_ltv as ( @@ -680,7 +680,7 @@ def test_snowflake_default_normalization(): ) -def test_snowflake_column_cast(): +def test_snowflake_column_cast() -> None: assert_sql_result( """ SELECT @@ -701,7 +701,7 @@ def test_snowflake_column_cast(): ) -def test_snowflake_unused_cte(): +def test_snowflake_unused_cte() -> None: # For this, we expect table level lineage to include table1, but CLL should not. assert_sql_result( """ @@ -723,7 +723,7 @@ def test_snowflake_unused_cte(): ) -def test_snowflake_cte_name_collision(): +def test_snowflake_cte_name_collision() -> None: # In this example, output col1 should come from table3 and not table1, since the cte is unused. # We'll still generate table-level lineage that includes table1. assert_sql_result( @@ -756,7 +756,7 @@ def test_snowflake_cte_name_collision(): ) -def test_snowflake_full_table_name_col_reference(): +def test_snowflake_full_table_name_col_reference() -> None: assert_sql_result( """ SELECT @@ -782,7 +782,7 @@ def test_snowflake_full_table_name_col_reference(): # TODO: Add a test for setting platform_instance or env -def test_teradata_default_normalization(): +def test_teradata_default_normalization() -> None: assert_sql_result( """ create table demo_user.test_lineage2 as @@ -824,7 +824,7 @@ def test_teradata_default_normalization(): ) -def test_teradata_strange_operators(): +def test_teradata_strange_operators() -> None: # This is a test for the following operators: # - `SEL` (select) # - `EQ` (equals) @@ -843,7 +843,7 @@ def test_teradata_strange_operators(): @pytest.mark.skip("sqlglot doesn't support this cast syntax yet") -def test_teradata_cast_syntax(): +def test_teradata_cast_syntax() -> None: assert_sql_result( """ SELECT my_table.date_col MONTH(4) AS month_col @@ -855,7 +855,7 @@ def test_teradata_cast_syntax(): ) -def test_snowflake_update_hardcoded(): +def test_snowflake_update_hardcoded() -> None: assert_sql_result( """ UPDATE snowflake_sample_data.tpch_sf1.orders @@ -873,7 +873,7 @@ def test_snowflake_update_hardcoded(): ) -def test_snowflake_update_from_table(): +def test_snowflake_update_from_table() -> None: # Can create these tables with the following SQL: """ -- Create or replace my_table @@ -946,7 +946,7 @@ def test_snowflake_update_from_table(): ) -def test_snowflake_update_self(): +def test_snowflake_update_self() -> None: assert_sql_result( """ UPDATE snowflake_sample_data.tpch_sf1.orders @@ -963,7 +963,7 @@ def test_snowflake_update_self(): ) -def test_postgres_select_subquery(): +def test_postgres_select_subquery() -> None: assert_sql_result( """ SELECT @@ -990,7 +990,7 @@ def test_postgres_select_subquery(): ) -def test_postgres_update_subselect(): +def test_postgres_update_subselect() -> None: assert_sql_result( """ UPDATE accounts SET sales_person_name = @@ -1016,7 +1016,7 @@ def test_postgres_update_subselect(): @pytest.mark.skip(reason="We can't parse column-list syntax with sub-selects yet") -def test_postgres_complex_update(): +def test_postgres_complex_update() -> None: # Example query from the postgres docs: # https://www.postgresql.org/docs/current/sql-update.html assert_sql_result( @@ -1043,7 +1043,7 @@ def test_postgres_complex_update(): ) -def test_redshift_materialized_view_auto_refresh(): +def test_redshift_materialized_view_auto_refresh() -> None: # Example query from the redshift docs: https://docs.aws.amazon.com/prescriptive-guidance/latest/materialized-views-redshift/refreshing-materialized-views.html assert_sql_result( """ @@ -1065,7 +1065,7 @@ def test_redshift_materialized_view_auto_refresh(): ) -def test_redshift_temp_table_shortcut(): +def test_redshift_temp_table_shortcut() -> None: # On redshift, tables starting with # are temporary tables. assert_sql_result( """ @@ -1092,7 +1092,7 @@ def test_redshift_temp_table_shortcut(): ) -def test_redshift_union_view(): +def test_redshift_union_view() -> None: # TODO: This currently fails to generate CLL. Need to debug further. assert_sql_result( """ @@ -1348,3 +1348,30 @@ def test_bigquery_indirect_references() -> None: }, }, ) + + +def test_oracle_case_insensitive_cols() -> None: + assert_sql_result( + """\ +SELECT +CASE WHEN (employee_TYPE = 'Manager') + THEN (CASE + WHEN job_title in ('Engineer') THEN '12' + ELSE '11' + END) + ELSE (CASE + WHEN job_title in ('Engineer') THEN '02' + ELSE '01' + END) +END AS employee_type_number +FROM ABC.employees ; +""", + dialect="oracle", + expected_file=RESOURCE_DIR / "test_oracle_case_insensitive_cols.json", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:oracle,abc.employees,PROD)": { + "employee_type": "VARCHAR2(100)", + "job_title": "VARCHAR2(100)", + }, + }, + )