Update test files with platform_instance

datahub-project · Feb 28, 2025 · f7276c1 · f7276c1
1 parent bfc3e34
commit f7276c1
Show file tree

Hide file tree

Showing 18 changed files with 1,857 additions and 738 deletions.
diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/basic_iolets.py
@@ -26,7 +26,9 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
-            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
+            Urn(
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.test_dag,PROD),test_task)"
+            ),
         ],
         outlets=[
             Dataset("snowflake", "mydb.schema.tableD"),

diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py
@@ -20,7 +20,9 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
-            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
+            Urn(
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.test_dag,PROD),test_task)"
+            ),
         ],
         outlets=[Dataset("snowflake", "mydb.schema.tableD")],
     )

diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/simple_dag.py
@@ -21,7 +21,9 @@
             Urn(
                 "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"
             ),
-            Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"),
+            Urn(
+                "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.test_dag,PROD),test_task)"
+            ),
         ],
         outlets=[Dataset("snowflake", "mydb.schema.tableD")],
     )

diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json
@@ -1,7 +1,7 @@
 [
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "dataFlowInfo",
     "aspect": {
@@ -25,7 +25,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -36,7 +36,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -60,7 +60,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "globalTags",
     "aspect": {
@@ -71,7 +71,7 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -82,7 +82,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -93,12 +93,16 @@
 },
 {
     "entityType": "dataFlow",
-    "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)",
+    "entityUrn": "urn:li:dataFlow:(airflow,myairflow.athena_operator,prod)",
     "changeType": "UPSERT",
     "aspectName": "browsePathsV2",
     "aspect": {
         "json": {
             "path": [
+                {
+                    "id": "myairflow",
+                    "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:airflow,myairflow)"
+                },
                 {
                     "id": "athena_operator"
                 }
@@ -108,7 +112,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -125,7 +129,7 @@
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
                 "outlets": "[]",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
             "name": "transform_cost_table",
@@ -138,7 +142,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -149,7 +153,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -250,7 +254,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -274,7 +278,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "globalTags",
     "aspect": {
@@ -302,16 +306,16 @@
                 "state": "running",
                 "operator": "AthenaOperator",
                 "priority_weight": "1",
-                "log_url": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
+                "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1",
                 "orchestrator": "airflow",
                 "dag_id": "athena_operator",
                 "task_id": "transform_cost_table"
             },
-            "externalUrl": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
+            "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1",
             "name": "athena_operator_transform_cost_table_manual_run_test",
             "type": "BATCH_AD_HOC",
             "created": {
-                "time": 1732719433576,
+                "time": 1740760883179,
                 "actor": "urn:li:corpuser:datahub"
             }
         }
@@ -324,7 +328,7 @@
     "aspectName": "dataProcessInstanceRelationships",
     "aspect": {
         "json": {
-            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+            "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
             "upstreamInstances": []
         }
     }
@@ -388,7 +392,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1732719433576,
+            "timestampMillis": 1740760883179,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
@@ -405,20 +409,20 @@
     "aspectName": "operation",
     "aspect": {
         "json": {
-            "timestampMillis": 1732719433736,
+            "timestampMillis": 1740760883509,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"
             },
             "actor": "urn:li:corpuser:airflow",
             "operationType": "CREATE",
-            "lastUpdatedTimestamp": 1732719433736
+            "lastUpdatedTimestamp": 1740760883509
         }
     }
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInfo",
     "aspect": {
@@ -435,7 +439,7 @@
                 "downstream_task_ids": "[]",
                 "inlets": "[]",
                 "outlets": "[]",
-                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
+                "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n        CREATE OR REPLACE TABLE processed_costs AS\\n        SELECT\\n            id,\\n            month,\\n            total_cost,\\n            area,\\n            total_cost / area as cost_per_area\\n        FROM costs\\n        \"}"
             },
             "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
             "name": "transform_cost_table",
@@ -448,7 +452,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -459,7 +463,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "dataJobInputOutput",
     "aspect": {
@@ -616,7 +620,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "ownership",
     "aspect": {
@@ -640,7 +644,7 @@
 },
 {
     "entityType": "dataJob",
-    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)",
+    "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,myairflow.athena_operator,prod),transform_cost_table)",
     "changeType": "UPSERT",
     "aspectName": "globalTags",
     "aspect": {
@@ -656,7 +660,7 @@
     "aspectName": "dataProcessInstanceRunEvent",
     "aspect": {
         "json": {
-            "timestampMillis": 1732719433747,
+            "timestampMillis": 1740760883541,
             "partitionSpec": {
                 "partition": "FULL_TABLE_SNAPSHOT",
                 "type": "FULL_TABLE"