updated golden test to cover physical upstream lineage

datahub-project · Mar 6, 2025 · 615a127 · 615a127
1 parent 64b12f7
commit 615a127
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 23 deletions.
diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
@@ -64,7 +64,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -122,7 +122,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -180,7 +180,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -238,7 +238,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -385,7 +385,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -532,7 +532,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -679,7 +679,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -826,7 +826,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -973,7 +973,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1120,7 +1120,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1267,7 +1267,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1414,7 +1414,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1561,7 +1561,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1578,7 +1578,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1595,7 +1595,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1612,14 +1612,14 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
 },
 {
-    "entityType": "chart",
-    "entityUrn": "urn:li:chart:(superset,13)",
+    "entityType": "tag",
+    "entityUrn": "urn:li:tag:superset:physical",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -1629,7 +1629,7 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }
@@ -1646,7 +1646,24 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "superset-2020_04_14-07_00_00-gqer6i",
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
+        "lastRunId": "no-run-id-provided",
+        "pipelineName": "test_pipeline"
+    }
+},
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(superset,13)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": true
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "superset-2020_04_14-07_00_00-o909wz",
         "lastRunId": "no-run-id-provided",
         "pipelineName": "test_pipeline"
     }

diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py
@@ -186,7 +186,6 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
                             }
                         ],
                         "schema": "test_schema1",
-                        "sql": "SELECT * FROM test_table1",
                         "table_name": "Test Table 1",
                     },
                     {
@@ -287,10 +286,8 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
                             "last_name": "Owner1",
                         }
                     ],
-                    "rendered_sql": "SELECT * FROM test_table1",
                     "schema": "test_schema1",
                     "select_star": "SELECT * FROM test_schema1.test_table1 LIMIT 100",
-                    "sql": "SELECT * FROM test_table1",
                     "table_name": "Test Table 1",
                     "uid": "1__table",
                     "url": "/tablemodelview/edit/1",