diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 883bc457c1b90c..9d4acd8829710c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -22,6 +22,7 @@ make_dataset_urn, make_dataset_urn_with_platform_instance, make_domain_urn, + make_user_urn, ) from datahub.emitter.mcp_builder import add_domain_to_entity_wu from datahub.ingestion.api.common import PipelineContext @@ -46,7 +47,6 @@ StatefulIngestionSourceBase, ) from datahub.metadata.com.linkedin.pegasus2avro.common import ( - AuditStamp, ChangeAuditStamps, Status, TimeStamp, @@ -65,12 +65,16 @@ SchemaMetadata, ) from datahub.metadata.schema_classes import ( + AuditStampClass, ChartInfoClass, ChartTypeClass, DashboardInfoClass, DatasetLineageTypeClass, DatasetPropertiesClass, GlobalTagsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, TagAssociationClass, UpstreamClass, UpstreamLineageClass, @@ -234,6 +238,7 @@ def __init__(self, ctx: PipelineContext, config: SupersetConfig): graph=self.ctx.graph, ) self.session = self.login() + self.owner_info = self.parse_owner_info() def login(self) -> requests.Session: login_response = requests.post( @@ -273,7 +278,7 @@ def paginate_entity_api_results(self, entity_type, page_size=100): while current_page * page_size < total_items: response = self.session.get( - f"{self.config.connect_uri}/api/v1/{entity_type}/", + f"{self.config.connect_uri}/api/v1/{entity_type}", params={"q": f"(page:{current_page},page_size:{page_size})"}, ) @@ -289,6 +294,25 @@ def paginate_entity_api_results(self, entity_type, page_size=100): current_page += 1 + def parse_owner_info(self) -> Dict[str, Any]: + entity_types = ["dataset", "dashboard", "chart"] + owners_info = {} + + for entity in entity_types: + for owner in self.paginate_entity_api_results(f"{entity}/related/owners"): + owner_id = owner.get("value") + if owner_id: + owners_info[owner_id] = owner.get("extra", {}).get("email", "") + + return owners_info + + def build_owner_urn(self, data: Dict[str, Any]) -> List[str]: + return [ + make_user_urn(self.owner_info.get(owner.get("id"), "")) + for owner in data.get("owners", []) + if owner.get("id") + ] + @lru_cache(maxsize=None) def get_dataset_info(self, dataset_id: int) -> dict: dataset_response = self.session.get( @@ -346,15 +370,16 @@ def construct_dashboard_from_api_data( aspects=[Status(removed=False)], ) - modified_actor = f"urn:li:corpuser:{(dashboard_data.get('changed_by') or {}).get('username', 'unknown')}" + modified_actor = f"urn:li:corpuser:{self.owner_info.get((dashboard_data.get('changed_by') or {}).get('id', -1), 'unknown')}" modified_ts = int( dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000 ) title = dashboard_data.get("dashboard_title", "") # note: the API does not currently supply created_by usernames due to a bug - last_modified = ChangeAuditStamps( - created=None, - lastModified=AuditStamp(time=modified_ts, actor=modified_actor), + last_modified = AuditStampClass(time=modified_ts, actor=modified_actor) + + change_audit_stamps = ChangeAuditStamps( + created=None, lastModified=last_modified ) dashboard_url = f"{self.config.display_uri}{dashboard_data.get('url', '')}" @@ -380,7 +405,7 @@ def construct_dashboard_from_api_data( "IsPublished": str(dashboard_data.get("published", False)).lower(), "Owners": ", ".join( map( - lambda owner: owner.get("username", "unknown"), + lambda owner: self.owner_info.get(owner.get("id", -1), "unknown"), dashboard_data.get("owners", []), ) ), @@ -400,15 +425,29 @@ def construct_dashboard_from_api_data( description="", title=title, charts=chart_urns, - lastModified=last_modified, dashboardUrl=dashboard_url, customProperties=custom_properties, + lastModified=change_audit_stamps, ) dashboard_snapshot.aspects.append(dashboard_info) + + dashboard_owners_list = self.build_owner_urn(dashboard_data) + owners_info = OwnershipClass( + owners=[ + OwnerClass( + owner=urn, + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + for urn in (dashboard_owners_list or []) + ], + lastModified=last_modified, + ) + dashboard_snapshot.aspects.append(owners_info) + return dashboard_snapshot def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]: - for dashboard_data in self.paginate_entity_api_results("dashboard", PAGE_SIZE): + for dashboard_data in self.paginate_entity_api_results("dashboard/", PAGE_SIZE): try: dashboard_snapshot = self.construct_dashboard_from_api_data( dashboard_data @@ -437,17 +476,19 @@ def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot: aspects=[Status(removed=False)], ) - modified_actor = f"urn:li:corpuser:{(chart_data.get('changed_by') or {}).get('username', 'unknown')}" + modified_actor = f"urn:li:corpuser:{self.owner_info.get((chart_data.get('changed_by') or {}).get('id', -1), 'unknown')}" modified_ts = int( dp.parse(chart_data.get("changed_on_utc", "now")).timestamp() * 1000 ) title = chart_data.get("slice_name", "") # note: the API does not currently supply created_by usernames due to a bug - last_modified = ChangeAuditStamps( - created=None, - lastModified=AuditStamp(time=modified_ts, actor=modified_actor), + last_modified = AuditStampClass(time=modified_ts, actor=modified_actor) + + change_audit_stamps = ChangeAuditStamps( + created=None, lastModified=last_modified ) + chart_type = chart_type_from_viz_type.get(chart_data.get("viz_type", "")) chart_url = f"{self.config.display_uri}{chart_data.get('url', '')}" @@ -504,16 +545,29 @@ def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot: type=chart_type, description="", title=title, - lastModified=last_modified, chartUrl=chart_url, inputs=[datasource_urn] if datasource_urn else None, customProperties=custom_properties, + lastModified=change_audit_stamps, ) chart_snapshot.aspects.append(chart_info) + + chart_owners_list = self.build_owner_urn(chart_data) + owners_info = OwnershipClass( + owners=[ + OwnerClass( + owner=urn, + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + for urn in (chart_owners_list or []) + ], + lastModified=last_modified, + ) + chart_snapshot.aspects.append(owners_info) return chart_snapshot def emit_chart_mces(self) -> Iterable[MetadataWorkUnit]: - for chart_data in self.paginate_entity_api_results("chart", PAGE_SIZE): + for chart_data in self.paginate_entity_api_results("chart/", PAGE_SIZE): try: chart_snapshot = self.construct_chart_from_chart_data(chart_data) @@ -583,6 +637,12 @@ def construct_dataset_from_dataset_data( ) dataset_url = f"{self.config.display_uri}{dataset_response.get('result', {}).get('url', '')}" + modified_actor = f"urn:li:corpuser:{self.owner_info.get((dataset_data.get('changed_by') or {}).get('id', -1), 'unknown')}" + modified_ts = int( + dp.parse(dataset_data.get("changed_on_utc", "now")).timestamp() * 1000 + ) + last_modified = AuditStampClass(time=modified_ts, actor=modified_actor) + upstream_warehouse_platform = ( dataset_response.get("result", {}).get("database", {}).get("backend") ) @@ -618,10 +678,8 @@ def construct_dataset_from_dataset_data( dataset_info = DatasetPropertiesClass( name=dataset.table_name, description="", - lastModified=( - TimeStamp(time=dataset.modified_ts) if dataset.modified_ts else None - ), externalUrl=dataset_url, + lastModified=TimeStamp(time=modified_ts), ) global_tags = GlobalTagsClass(tags=[TagAssociationClass(tag=tag_urn)]) @@ -640,12 +698,23 @@ def construct_dataset_from_dataset_data( aspects=aspects_items, ) - logger.info(f"Constructed dataset {datasource_urn}") + dataset_owners_list = self.build_owner_urn(dataset_data) + owners_info = OwnershipClass( + owners=[ + OwnerClass( + owner=urn, + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + for urn in (dataset_owners_list or []) + ], + lastModified=last_modified, + ) + aspects_items.append(owners_info) return dataset_snapshot def emit_dataset_mces(self) -> Iterable[MetadataWorkUnit]: - for dataset_data in self.paginate_entity_api_results("dataset", PAGE_SIZE): + for dataset_data in self.paginate_entity_api_results("dataset/", PAGE_SIZE): try: dataset_snapshot = self.construct_dataset_from_dataset_data( dataset_data diff --git a/metadata-ingestion/tests/integration/preset/golden_test_ingest.json b/metadata-ingestion/tests/integration/preset/golden_test_ingest.json index cb0535ecd1483e..7db59082901829 100644 --- a/metadata-ingestion/tests/integration/preset/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/preset/golden_test_ingest.json @@ -14,7 +14,7 @@ "customProperties": { "Status": "published", "IsPublished": "true", - "Owners": "test_username_1, test_username_2", + "Owners": "test_owner1@example.com, test_owner2@example.com", "IsCertified": "true", "CertifiedBy": "Certification team", "CertificationDetails": "Approved" @@ -34,11 +34,30 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "dashboardUrl": "mock://mock-domain.preset.io/dashboard/test_dashboard_url_1" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner1@example.com", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -64,7 +83,7 @@ "customProperties": { "Status": "draft", "IsPublished": "false", - "Owners": "unknown", + "Owners": "test_owner2@example.com", "IsCertified": "false" }, "title": "test_dashboard_title_2", @@ -82,11 +101,26 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "dashboardUrl": "mock://mock-domain.preset.io/dashboard/test_dashboard_url_2" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } @@ -123,7 +157,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_10", @@ -134,6 +168,16 @@ ], "type": "BAR" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -170,7 +214,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_11", @@ -181,6 +225,16 @@ ], "type": "PIE" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -217,7 +271,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_12", @@ -228,6 +282,16 @@ ], "type": "AREA" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } @@ -264,7 +328,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_13", @@ -275,6 +339,16 @@ ], "type": "HISTOGRAM" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } diff --git a/metadata-ingestion/tests/integration/preset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/preset/golden_test_stateful_ingest.json index 1f0d1310b77bb8..514dc08dacab01 100644 --- a/metadata-ingestion/tests/integration/preset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/preset/golden_test_stateful_ingest.json @@ -14,7 +14,7 @@ "customProperties": { "Status": "published", "IsPublished": "true", - "Owners": "test_username_1, test_username_2", + "Owners": "test_owner1@example.com, test_owner2@example.com", "IsCertified": "true", "CertifiedBy": "Certification team", "CertificationDetails": "Approved" @@ -34,18 +34,37 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "dashboardUrl": "mock://mock-domain.preset.io/dashboard/test_dashboard_url_1" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner1@example.com", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -76,7 +95,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_10", @@ -87,13 +106,23 @@ ], "type": "BAR" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -124,7 +153,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_11", @@ -135,13 +164,23 @@ ], "type": "PIE" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -172,7 +211,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_12", @@ -183,13 +222,23 @@ ], "type": "AREA" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -220,7 +269,7 @@ }, "lastModified": { "time": 1720594800000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.preset.io/explore/test_chart_url_13", @@ -231,13 +280,23 @@ ], "type": "HISTOGRAM" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1720594800000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -254,7 +313,7 @@ }, "systemMetadata": { "lastObserved": 1720594800000, - "runId": "preset-2024_07_10-07_00_00-xe6j8e", + "runId": "preset-2024_07_10-07_00_00-xazuyh", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } diff --git a/metadata-ingestion/tests/integration/preset/test_preset.py b/metadata-ingestion/tests/integration/preset/test_preset.py index f926a762e6a078..2620a582937a28 100644 --- a/metadata-ingestion/tests/integration/preset/test_preset.py +++ b/metadata-ingestion/tests/integration/preset/test_preset.py @@ -60,7 +60,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "1", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_1", @@ -70,10 +72,14 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "published": True, "owners": [ { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, ], "certified_by": "Certification team", @@ -82,7 +88,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "2", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_2", @@ -92,8 +100,10 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "published": False, "owners": [ { - "first_name": "name", - }, + "first_name": "Test", + "id": 2, + "last_name": "Owner2", + } ], "certified_by": "", "certification_details": "", @@ -110,7 +120,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "10", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "slice_name": "test_chart_title_1", @@ -122,7 +134,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "11", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "slice_name": "test_chart_title_2", @@ -134,7 +148,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "12", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "slice_name": "test_chart_title_3", @@ -146,7 +162,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "13", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "slice_name": "test_chart_title_4", @@ -181,6 +199,63 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - }, }, }, + "mock://mock-domain.preset.io/api/v1/dashboard/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner1@example.com"}, + "text": "test_owner1", + "value": 1, + }, + { + "extra": {"active": True, "email": "test_owner2@example.com"}, + "text": "test_owner2", + "value": 2, + }, + ], + }, + }, + "mock://mock-domain.preset.io/api/v1/dataset/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner3@example.com"}, + "text": "test_owner3", + "value": 3, + }, + { + "extra": {"active": True, "email": "test_owner4@example.com"}, + "text": "test_owner4", + "value": 4, + }, + ], + }, + }, + "mock://mock-domain.preset.io/api/v1/chart/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner5@example.com"}, + "text": "test_owner5", + "value": 5, + }, + { + "extra": {"active": True, "email": "test_owner6@example.com"}, + "text": "test_owner6", + "value": 6, + }, + ], + }, + }, } api_vs_response.update(override_data) @@ -281,7 +356,9 @@ def test_preset_stateful_ingest( { "id": "1", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, "changed_on_utc": "2024-07-10T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_1", @@ -291,10 +368,14 @@ def test_preset_stateful_ingest( "published": True, "owners": [ { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owners2", }, ], "certified_by": "Certification team", diff --git a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json index 324f32da12059e..3be7c28cc52a66 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json @@ -14,7 +14,7 @@ "customProperties": { "Status": "published", "IsPublished": "true", - "Owners": "test_username_1, test_username_2", + "Owners": "test_owner1@example.com, test_owner2@example.com", "IsCertified": "true", "CertifiedBy": "Certification team", "CertificationDetails": "Approved" @@ -34,11 +34,30 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "dashboardUrl": "mock://mock-domain.superset.com/dashboard/test_dashboard_url_1" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner1@example.com", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -64,7 +83,7 @@ "customProperties": { "Status": "draft", "IsPublished": "false", - "Owners": "unknown", + "Owners": "test_owner4@example.com", "IsCertified": "false" }, "title": "test_dashboard_title_2", @@ -82,11 +101,26 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "dashboardUrl": "mock://mock-domain.superset.com/dashboard/test_dashboard_url_2" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner4@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } @@ -123,7 +157,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_10", @@ -134,6 +168,16 @@ ], "type": "BAR" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -170,7 +214,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_11", @@ -181,6 +225,16 @@ ], "type": "PIE" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } @@ -217,7 +271,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_12", @@ -228,6 +282,16 @@ ], "type": "AREA" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } @@ -264,7 +328,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_13", @@ -275,6 +339,16 @@ ], "type": "HISTOGRAM" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json index 01e28b3368ce9f..772c58d6ca6170 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json @@ -14,7 +14,7 @@ "customProperties": { "Status": "published", "IsPublished": "true", - "Owners": "test_username_1, test_username_2", + "Owners": "test_owner1@example.com, test_owner2@example.com", "IsCertified": "true", "CertifiedBy": "Certification team", "CertificationDetails": "Approved" @@ -34,18 +34,37 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "dashboardUrl": "mock://mock-domain.superset.com/dashboard/test_dashboard_url_1" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner1@example.com", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -76,7 +95,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_10", @@ -87,13 +106,23 @@ ], "type": "BAR" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -124,7 +153,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "actor": "urn:li:corpuser:test_owner1@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_11", @@ -135,13 +164,23 @@ ], "type": "PIE" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner1@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -172,7 +211,7 @@ }, "lastModified": { "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "actor": "urn:li:corpuser:test_owner2@example.com" } }, "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_12", @@ -183,13 +222,23 @@ ], "type": "AREA" } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -227,6 +276,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -255,13 +307,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -299,6 +366,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -327,13 +397,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -371,6 +456,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -399,13 +487,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -443,6 +546,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -471,13 +577,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -515,6 +636,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -543,13 +667,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -587,6 +726,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -615,13 +757,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -659,6 +816,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -687,13 +847,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -731,6 +906,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -759,13 +937,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -803,6 +996,9 @@ "externalUrl": "mock://mock-domain.superset.com/tablemodelview/edit/2", "name": "Test Table 2", "description": "", + "lastModified": { + "time": 1707579020123 + }, "tags": [] } }, @@ -831,13 +1027,28 @@ } ] } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test_owner2@example.com", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1707579020123, + "actor": "urn:li:corpuser:test_owner2@example.com" + } + } } ] } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -854,7 +1065,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -871,7 +1082,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -888,14 +1099,14 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(superset,2)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(superset,13)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -905,14 +1116,14 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(superset,13)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(superset,2)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -922,7 +1133,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00-daaqpe", + "runId": "superset-2020_04_14-07_00_00-ptptj3", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py index a9246bc8953f5b..7bee6e5bcf6aa5 100644 --- a/metadata-ingestion/tests/integration/superset/test_superset.py +++ b/metadata-ingestion/tests/integration/superset/test_superset.py @@ -37,7 +37,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "1", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_1", @@ -47,10 +49,14 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "published": True, "owners": [ { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, ], "certified_by": "Certification team", @@ -59,7 +65,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "2", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owners2", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_2", @@ -69,8 +77,10 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "published": False, "owners": [ { - "first_name": "name", - }, + "first_name": "Test", + "id": 4, + "last_name": "Owner4", + } ], "certified_by": "", "certification_details": "", @@ -87,7 +97,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "10", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_1", @@ -99,7 +111,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "11", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_2", @@ -111,7 +125,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "12", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owners2", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_3", @@ -123,7 +139,9 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "13", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owners2", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_4", @@ -148,7 +166,6 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "first_name": "Test", "id": 1, "last_name": "User1", - "username": "test_username_1", }, "changed_by_name": "test_username_1", "changed_on_delta_humanized": "10 months ago", @@ -166,7 +183,6 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "first_name": "Test", "id": 1, "last_name": "Owner1", - "username": "test_username_1", } ], "schema": "test_schema1", @@ -178,7 +194,6 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "first_name": "Test", "id": 2, "last_name": "User2", - "username": "test_username_2", }, "changed_by_name": "test_username_2", "changed_on_delta_humanized": "9 months ago", @@ -196,7 +211,6 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "first_name": "Test", "id": 2, "last_name": "Owner2", - "username": "test_username_2", } ], "schema": "test_schema2", @@ -214,7 +228,11 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "result": { "always_filter_main_dttm": False, "cache_timeout": None, - "changed_by": {"first_name": "Test", "last_name": "User1"}, + "changed_by": { + "first_name": "Test", + "id": 1, + "last_name": "Owners1", + }, "changed_on": "2024-01-05T21:10:15.650819+0000", "changed_on_humanized": "10 months ago", "created_by": {"first_name": "Test", "last_name": "User1"}, @@ -262,7 +280,13 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "name": "Test Table 1", "normalize_columns": True, "offset": 0, - "owners": [{"first_name": "Test", "id": 1, "last_name": "Owner1"}], + "owners": [ + { + "first_name": "Test", + "id": 1, + "last_name": "Owner1", + } + ], "rendered_sql": "SELECT * FROM test_table1", "schema": "test_schema1", "select_star": "SELECT * FROM test_schema1.test_table1 LIMIT 100", @@ -288,7 +312,11 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "result": { "always_filter_main_dttm": False, "cache_timeout": None, - "changed_by": {"first_name": "Test", "last_name": "User2"}, + "changed_by": { + "first_name": "Test", + "id": 2, + "last_name": "Owners2", + }, "changed_on": "2024-02-10T15:30:20.123456+0000", "changed_on_humanized": "9 months ago", "created_by": {"first_name": "Test", "last_name": "User2"}, @@ -333,7 +361,13 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "name": "Test Table 2", "normalize_columns": True, "offset": 0, - "owners": [{"first_name": "Test", "id": 2, "last_name": "Owner2"}], + "owners": [ + { + "first_name": "Test", + "id": 2, + "last_name": "Owner2", + } + ], "rendered_sql": "SELECT * FROM test_table2", "schema": "test_schema2", "select_star": "SELECT * FROM test_schema2.test_table2 LIMIT 100", @@ -387,6 +421,63 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - }, }, }, + "mock://mock-domain.superset.com/api/v1/dashboard/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner1@example.com"}, + "text": "test_owner1", + "value": 1, + }, + { + "extra": {"active": True, "email": "test_owner2@example.com"}, + "text": "test_owner2", + "value": 2, + }, + ], + }, + }, + "mock://mock-domain.superset.com/api/v1/dataset/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner3@example.com"}, + "text": "test_owner3", + "value": 3, + }, + { + "extra": {"active": True, "email": "test_owner4@example.com"}, + "text": "test_owner4", + "value": 4, + }, + ], + }, + }, + "mock://mock-domain.superset.com/api/v1/chart/related/owners": { + "method": "GET", + "status_code": 200, + "json": { + "count": 2, + "result": [ + { + "extra": {"active": True, "email": "test_owner5@example.com"}, + "text": "test_owner5", + "value": 5, + }, + { + "extra": {"active": True, "email": "test_owner6@example.com"}, + "text": "test_owner6", + "value": 6, + }, + ], + }, + }, } api_vs_response.update(override_data) @@ -487,7 +578,9 @@ def test_superset_stateful_ingest( { "id": "1", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "dashboard_title": "test_dashboard_title_1", @@ -497,10 +590,14 @@ def test_superset_stateful_ingest( "published": True, "owners": [ { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owner1", }, { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owner2", }, ], "certified_by": "Certification team", @@ -518,7 +615,9 @@ def test_superset_stateful_ingest( { "id": "10", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_1", @@ -530,7 +629,9 @@ def test_superset_stateful_ingest( { "id": "11", "changed_by": { - "username": "test_username_1", + "first_name": "Test", + "id": 1, + "last_name": "Owners1", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_2", @@ -542,7 +643,9 @@ def test_superset_stateful_ingest( { "id": "12", "changed_by": { - "username": "test_username_2", + "first_name": "Test", + "id": 2, + "last_name": "Owners2", }, "changed_on_utc": "2020-04-14T07:00:00.000000+0000", "slice_name": "test_chart_title_3", @@ -567,7 +670,6 @@ def test_superset_stateful_ingest( "first_name": "Test", "id": 2, "last_name": "User2", - "username": "test_username_2", }, "changed_by_name": "test_username_2", "changed_on_delta_humanized": "9 months ago", @@ -585,7 +687,6 @@ def test_superset_stateful_ingest( "first_name": "Test", "id": 2, "last_name": "Owner2", - "username": "test_username_2", } ], "schema": "test_schema2", diff --git a/metadata-ingestion/tests/unit/test_superset_source.py b/metadata-ingestion/tests/unit/test_superset_source.py index 912bfa3511421c..dc0c2fce273bf2 100644 --- a/metadata-ingestion/tests/unit/test_superset_source.py +++ b/metadata-ingestion/tests/unit/test_superset_source.py @@ -1,4 +1,5 @@ -from datahub.ingestion.source.superset import SupersetConfig +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.superset import SupersetConfig, SupersetSource def test_default_values(): @@ -19,3 +20,61 @@ def test_set_display_uri(): assert config.connect_uri == "http://localhost:8088" assert config.display_uri == display_uri + + +def test_superset_login(requests_mock): + login_url = "http://localhost:8088/api/v1/security/login" + requests_mock.post(login_url, json={"access_token": "dummy_token"}, status_code=200) + + dashboard_url = "http://localhost:8088/api/v1/dashboard/" + requests_mock.get(dashboard_url, json={}, status_code=200) + + for entity in ["dataset", "dashboard", "chart"]: + requests_mock.get( + f"http://localhost:8088/api/v1/{entity}/related/owners", + json={}, + status_code=200, + ) + + source = SupersetSource( + ctx=PipelineContext(run_id="superset-source-test"), config=SupersetConfig() + ) + assert source.platform == "superset" + + +def test_superset_build_owners_info(requests_mock): + login_url = "http://localhost:8088/api/v1/security/login" + requests_mock.post(login_url, json={"access_token": "dummy_token"}, status_code=200) + + dashboard_url = "http://localhost:8088/api/v1/dashboard/" + requests_mock.get(dashboard_url, json={}, status_code=200) + + for entity in ["dataset", "dashboard", "chart"]: + requests_mock.get( + f"http://localhost:8088/api/v1/{entity}/related/owners", + json={ + "count": 2, + "result": [ + { + "extra": {"active": "false", "email": "test_user1@example.com"}, + "text": "Test User1", + "value": 1, + }, + { + "extra": {"active": "false", "email": "test_user2@example.com"}, + "text": "Test User2", + "value": 2, + }, + ], + }, + status_code=200, + ) + + source = SupersetSource( + ctx=PipelineContext(run_id="superset-source-owner-info-test"), + config=SupersetConfig(), + ) + assert source.owner_info == { + 1: "test_user1@example.com", + 2: "test_user2@example.com", + }