From 22403590773c69fa37a8c4a4f63c7e6e9de63fd2 Mon Sep 17 00:00:00 2001 From: Aryamanz29 Date: Wed, 3 Apr 2024 15:48:40 +0530 Subject: [PATCH 1/2] DVX-358: Fixes issues with `AtlanEvent` model deserialization --- pyatlan/model/events.py | 9 +- tests/unit/data/event_responses/actual.json | 40 ++ .../business_attribute_update.json | 84 ++++ .../event_responses/classification_add.json | 113 ++++++ .../classification_delete.json | 100 +++++ .../data/event_responses/entity_create.json | 90 +++++ .../data/event_responses/entity_delete.json | 36 ++ .../data/event_responses/entity_update.json | 125 ++++++ .../unit/data/event_responses/validation.json | 39 ++ tests/unit/test_events.py | 366 ++++++------------ 10 files changed, 745 insertions(+), 257 deletions(-) create mode 100644 tests/unit/data/event_responses/actual.json create mode 100644 tests/unit/data/event_responses/business_attribute_update.json create mode 100644 tests/unit/data/event_responses/classification_add.json create mode 100644 tests/unit/data/event_responses/classification_delete.json create mode 100644 tests/unit/data/event_responses/entity_create.json create mode 100644 tests/unit/data/event_responses/entity_delete.json create mode 100644 tests/unit/data/event_responses/entity_update.json create mode 100644 tests/unit/data/event_responses/validation.json diff --git a/pyatlan/model/events.py b/pyatlan/model/events.py index 1b7dbdc5e..e8a41336a 100644 --- a/pyatlan/model/events.py +++ b/pyatlan/model/events.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright 2023 Atlan Pte. Ltd. -from typing import Any, Dict, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic.v1 import Field @@ -84,7 +84,7 @@ class AtlanTagAddPayload( operation_type: Literal["CLASSIFICATION_ADD"] = Field( description="Type of the operation the event contains a payload for." ) - mutated_details: Optional[AtlanTag] = Field( + mutated_details: Optional[List[AtlanTag]] = Field( description="Atlan tags that were added to the asset by this event." ) @@ -95,7 +95,7 @@ class AtlanTagDeletePayload( operation_type: Literal["CLASSIFICATION_DELETE"] = Field( description="Type of the operation the event contains a payload for." ) - mutated_details: Optional[AtlanTag] = Field( + mutated_details: Optional[List[AtlanTag]] = Field( description="Atlan tags that were removed from the asset by this event." ) @@ -118,9 +118,10 @@ class AtlanEvent(AtlanObject): Union[ AssetCreatePayload, AssetUpdatePayload, - CustomMetadataUpdatePayload, + AssetDeletePayload, AtlanTagAddPayload, AtlanTagDeletePayload, + CustomMetadataUpdatePayload, ] ] = Field( description="Detailed contents (payload) of the event.", diff --git a/tests/unit/data/event_responses/actual.json b/tests/unit/data/event_responses/actual.json new file mode 100644 index 000000000..8afd05839 --- /dev/null +++ b/tests/unit/data/event_responses/actual.json @@ -0,0 +1,40 @@ +{ + "version": "2.0", + "routeKey": "$default", + "rawPath": "/", + "rawQueryString": "", + "headers": { + "content-length": "85", + "x-atlan-signing-secret": "test-secret", + "x-amzn-tls-cipher-suite": "ECDHE-RSA-AES128-GCM-SHA256", + "x-amzn-tls-version": "TLSv1.2", + "x-amzn-trace-id": "Root=1-64931f85-02856e0b115997577ed16d97", + "x-forwarded-proto": "https", + "host": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", + "x-forwarded-port": "443", + "content-type": "text/plain; charset=utf-8", + "x-forwarded-for": "34.194.9.164", + "accept-encoding": "gzip", + "user-agent": "go-resty/1.12.0 (https://github.com/go-resty/resty)" + }, + "requestContext": { + "accountId": "anonymous", + "apiId": "uo8rokyifhnhubgagp106xhrpvgmuqhx", + "domainName": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", + "domainPrefix": "uo8rokyifhnhubgagp106xhrpvgmuqhx", + "http": { + "method": "POST", + "path": "/", + "protocol": "HTTP/1.1", + "sourceIp": "34.194.9.164", + "userAgent": "go-resty/1.12.0 (https://github.com/go-resty/resty)" + }, + "requestId": "27539f31-d444-419e-b1ad-4382657b7e04", + "routeKey": "$default", + "stage": "$default", + "time": "21/Jun/2023:16:04:21 +0000", + "timeEpoch": 1687363461544 + }, + "body": "{\"source\":{},\"version\":{\"version\":\"1.0.0\",\"versionParts\":[1]},\"msgCompressionKind\":\"NONE\",\"msgSplitIdx\":1,\"msgSplitCount\":1,\"msgSourceIP\":\"10.121.193.228\",\"msgCreatedBy\":\"\",\"msgCreationTime\":1666792334952,\"spooled\":false,\"message\":{\"type\":\"ENTITY_NOTIFICATION_V2\",\"entity\":{\"typeName\":\"AtlasGlossaryTerm\",\"attributes\":{\"qualifiedName\":\"rrSsMGHn1q1W2dWHqSHWe@Q07rYkHOACJkmUZJgmKFP\",\"name\":\"Example\"},\"guid\":\"bacab52b-6c4b-4dbe-b5da-97ec2e509c7e\",\"status\":\"ACTIVE\",\"displayText\":\"Example\"},\"operationType\":\"ENTITY_CREATE\",\"eventTime\":1666792332986}}", + "isBase64Encoded": false +} diff --git a/tests/unit/data/event_responses/business_attribute_update.json b/tests/unit/data/event_responses/business_attribute_update.json new file mode 100644 index 000000000..b8573b248 --- /dev/null +++ b/tests/unit/data/event_responses/business_attribute_update.json @@ -0,0 +1,84 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.121.193.228", + "msgCreatedBy": "", + "msgCreationTime": 1667822461771, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "Table", + "attributes": { + "popularityScore": 1.17549435e-38, + "lastSyncWorkflowName": "mabl-qa-006-crawler", + "lastSyncRunAt": 1665132063880, + "databaseName": "atlan_trial", + "connectionQualifiedName": "default/postgres/1665131987", + "assetDbtJobLastRunHasSourcesGenerated": false, + "assetDbtJobLastRunHasDocsGenerated": false, + "queryCount": 0, + "isTemporary": false, + "lastSyncRun": "atlan-postgres-1665131987-k9cds", + "isPartitioned": false, + "schemaName": "demo", + "adminUsers": [], + "partitionCount": 0, + "queryUserCount": 0, + "assetDbtJobLastRunUpdatedAt": 0, + "ownerGroups": [], + "sourceUpdatedAt": 0, + "assetDbtJobLastRunArtifactsSaved": false, + "isEditable": true, + "announcementUpdatedAt": 0, + "assetDbtJobLastRunStartedAt": 0, + "isDiscoverable": true, + "rowCount": 134, + "isQueryPreview": true, + "schemaQualifiedName": "default/postgres/1665131987/atlan_trial/demo", + "sourceCreatedAt": 0, + "assetDbtJobLastRunDequedAt": 0, + "viewerUsers": [], + "assetDbtTags": [], + "adminRoles": [], + "adminGroups": [], + "assetDbtJobLastRunCreatedAt": 0, + "qualifiedName": "default/postgres/1665131987/atlan_trial/demo/INSTACART_AISLES", + "databaseQualifiedName": "default/postgres/1665131987/atlan_trial", + "assetDbtJobNextRun": 0, + "assetDbtJobLastRunNotificationsSent": false, + "columnCount": 2, + "sizeBytes": 0, + "name": "INSTACART_AISLES", + "certificateUpdatedAt": 0, + "connectorName": "postgres", + "viewerGroups": [], + "assetDbtJobLastRun": 0, + "ownerUsers": [] + }, + "guid": "31ee0651-b031-48c8-8927-ef9b1e949ad4", + "status": "ACTIVE", + "displayText": "INSTACART_AISLES", + "isIncomplete": false, + "createdBy": "autoqa", + "updatedBy": "autoqa", + "createTime": 1665132167750, + "updateTime": 1665132175423 + }, + "operationType": "BUSINESS_ATTRIBUTE_UPDATE", + "eventTime": 1667822461710, + "mutatedDetails": { + "UuPz0CjWHFuuKTCiTRwapk": { + "jpf1oz5ux6MjmaOGPn9jMD": "ssjmdvddf\n" + } + } + } +} diff --git a/tests/unit/data/event_responses/classification_add.json b/tests/unit/data/event_responses/classification_add.json new file mode 100644 index 000000000..d2cd978a2 --- /dev/null +++ b/tests/unit/data/event_responses/classification_add.json @@ -0,0 +1,113 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.121.193.228", + "msgCreatedBy": "", + "msgCreationTime": 1666964451608, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "Column", + "attributes": { + "popularityScore": 1.17549435e-38, + "lastSyncRunAt": 1666792022425, + "databaseName": "FIVETRAN", + "precision": 18, + "lastSyncRun": "atlan-snowflake-1666769582-55jct", + "schemaName": "POSTGRES_RDS_DEMO_DEV", + "tableName": "INSTACART_UNIQUE_USERS", + "adminUsers": [], + "queryUserCount": 0, + "sourceUpdatedAt": 0, + "assetDbtJobLastRunArtifactsSaved": false, + "isEditable": true, + "isPartition": false, + "announcementUpdatedAt": 0, + "order": 1, + "sourceCreatedAt": 0, + "assetDbtJobLastRunDequedAt": 0, + "assetDbtTags": [], + "isIndexed": false, + "qualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV/INSTACART_UNIQUE_USERS/TOTAL_USER", + "dataType": "NUMBER", + "assetDbtJobLastRunNotificationsSent": false, + "isClustered": false, + "isNullable": false, + "name": "TOTAL_USER", + "certificateUpdatedAt": 0, + "connectorName": "snowflake", + "numericScale": 0, + "maxLength": 0, + "ownerUsers": [], + "isSort": false, + "lastSyncWorkflowName": "activate-poc-crawler", + "connectionQualifiedName": "default/snowflake/1666769582", + "assetDbtJobLastRunHasSourcesGenerated": false, + "isForeign": false, + "assetDbtJobLastRunHasDocsGenerated": false, + "queryCount": 0, + "pinnedAt": 0, + "isDist": false, + "assetDbtJobLastRunUpdatedAt": 0, + "ownerGroups": [], + "isPrimary": false, + "assetDbtJobLastRunStartedAt": 0, + "isDiscoverable": true, + "schemaQualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV", + "viewerUsers": [], + "adminRoles": [], + "adminGroups": [], + "isPinned": false, + "assetDbtJobLastRunCreatedAt": 0, + "databaseQualifiedName": "default/snowflake/1666769582/FIVETRAN", + "assetDbtJobNextRun": 0, + "tableQualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV/INSTACART_UNIQUE_USERS", + "partitionOrder": 0, + "viewerGroups": [], + "assetDbtJobLastRun": 0 + }, + "guid": "9065bba5-22b8-4331-bff8-1bdf017c5cfb", + "status": "ACTIVE", + "displayText": "TOTAL_USER", + "classificationNames": [ + "jUQpLPdg9hWoheH2eL9Ha2" + ], + "classifications": [ + { + "typeName": "jUQpLPdg9hWoheH2eL9Ha2", + "entityGuid": "9065bba5-22b8-4331-bff8-1bdf017c5cfb", + "entityStatus": "ACTIVE", + "propagate": true, + "removePropagationsOnEntityDelete": true, + "restrictPropagationThroughLineage": false + } + ], + "isIncomplete": false, + "createdBy": "kartik.thakur", + "updatedBy": "kartik.thakur", + "createTime": 1666792644294, + "updateTime": 1666964451417 + }, + "operationType": "CLASSIFICATION_ADD", + "eventTime": 1666964451417, + "mutatedDetails": [ + { + "typeName": "jUQpLPdg9hWoheH2eL9Ha2", + "entityGuid": "9065bba5-22b8-4331-bff8-1bdf017c5cfb", + "entityStatus": "ACTIVE", + "propagate": true, + "removePropagationsOnEntityDelete": true, + "restrictPropagationThroughLineage": false + } + ] + } +} diff --git a/tests/unit/data/event_responses/classification_delete.json b/tests/unit/data/event_responses/classification_delete.json new file mode 100644 index 000000000..5e381c118 --- /dev/null +++ b/tests/unit/data/event_responses/classification_delete.json @@ -0,0 +1,100 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.121.193.228", + "msgCreatedBy": "", + "msgCreationTime": 1666964493302, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "Column", + "attributes": { + "popularityScore": 1.17549435e-38, + "lastSyncRunAt": 1666792022425, + "databaseName": "FIVETRAN", + "precision": 18, + "lastSyncRun": "atlan-snowflake-1666769582-55jct", + "schemaName": "POSTGRES_RDS_DEMO_DEV", + "tableName": "INSTACART_UNIQUE_USERS", + "adminUsers": [], + "queryUserCount": 0, + "sourceUpdatedAt": 0, + "assetDbtJobLastRunArtifactsSaved": false, + "isEditable": true, + "isPartition": false, + "announcementUpdatedAt": 0, + "order": 1, + "sourceCreatedAt": 0, + "assetDbtJobLastRunDequedAt": 0, + "assetDbtTags": [], + "isIndexed": false, + "qualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV/INSTACART_UNIQUE_USERS/TOTAL_USER", + "dataType": "NUMBER", + "assetDbtJobLastRunNotificationsSent": false, + "isClustered": false, + "isNullable": false, + "name": "TOTAL_USER", + "certificateUpdatedAt": 0, + "connectorName": "snowflake", + "numericScale": 0, + "maxLength": 0, + "ownerUsers": [], + "isSort": false, + "lastSyncWorkflowName": "activate-poc-crawler", + "connectionQualifiedName": "default/snowflake/1666769582", + "assetDbtJobLastRunHasSourcesGenerated": false, + "isForeign": false, + "assetDbtJobLastRunHasDocsGenerated": false, + "queryCount": 0, + "pinnedAt": 0, + "isDist": false, + "assetDbtJobLastRunUpdatedAt": 0, + "ownerGroups": [], + "isPrimary": false, + "assetDbtJobLastRunStartedAt": 0, + "isDiscoverable": true, + "schemaQualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV", + "viewerUsers": [], + "adminRoles": [], + "adminGroups": [], + "isPinned": false, + "assetDbtJobLastRunCreatedAt": 0, + "databaseQualifiedName": "default/snowflake/1666769582/FIVETRAN", + "assetDbtJobNextRun": 0, + "tableQualifiedName": "default/snowflake/1666769582/FIVETRAN/POSTGRES_RDS_DEMO_DEV/INSTACART_UNIQUE_USERS", + "partitionOrder": 0, + "viewerGroups": [], + "assetDbtJobLastRun": 0 + }, + "guid": "9065bba5-22b8-4331-bff8-1bdf017c5cfb", + "status": "ACTIVE", + "displayText": "TOTAL_USER", + "isIncomplete": false, + "createdBy": "kartik.thakur", + "updatedBy": "kartik.thakur", + "createTime": 1666792644294, + "updateTime": 1666964493133 + }, + "operationType": "CLASSIFICATION_DELETE", + "eventTime": 1666964493133, + "mutatedDetails": [ + { + "typeName": "jUQpLPdg9hWoheH2eL9Ha2", + "entityGuid": "9065bba5-22b8-4331-bff8-1bdf017c5cfb", + "entityStatus": "ACTIVE", + "propagate": true, + "removePropagationsOnEntityDelete": true, + "restrictPropagationThroughLineage": false + } + ] + } +} diff --git a/tests/unit/data/event_responses/entity_create.json b/tests/unit/data/event_responses/entity_create.json new file mode 100644 index 000000000..cd86b0baf --- /dev/null +++ b/tests/unit/data/event_responses/entity_create.json @@ -0,0 +1,90 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.121.193.228", + "msgCreatedBy": "", + "msgCreationTime": 1666792334952, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "Column", + "attributes": { + "popularityScore": 1.17549435e-38, + "lastSyncRunAt": 1666792022425, + "databaseName": "FIVETRAN", + "precision": 0, + "lastSyncRun": "atlan-snowflake-1666769582-55jct", + "schemaName": "DEMO_SALESFORCE", + "tableName": "DASHBOARD_FEED", + "adminUsers": [], + "queryUserCount": 0, + "sourceUpdatedAt": 0, + "assetDbtJobLastRunArtifactsSaved": false, + "isEditable": true, + "isPartition": false, + "announcementUpdatedAt": 0, + "order": 6, + "sourceCreatedAt": 0, + "assetDbtJobLastRunDequedAt": 0, + "assetDbtTags": [], + "isIndexed": false, + "qualifiedName": "default/snowflake/1666769582/FIVETRAN/DEMO_SALESFORCE/DASHBOARD_FEED/IS_DELETED", + "dataType": "BOOLEAN", + "assetDbtJobLastRunNotificationsSent": false, + "isClustered": false, + "isNullable": false, + "name": "IS_DELETED", + "certificateUpdatedAt": 0, + "connectorName": "snowflake", + "numericScale": 0, + "maxLength": 0, + "ownerUsers": [], + "isSort": false, + "lastSyncWorkflowName": "activate-poc-crawler", + "connectionQualifiedName": "default/snowflake/1666769582", + "assetDbtJobLastRunHasSourcesGenerated": false, + "isForeign": false, + "assetDbtJobLastRunHasDocsGenerated": false, + "queryCount": 0, + "pinnedAt": 0, + "isDist": false, + "assetDbtJobLastRunUpdatedAt": 0, + "ownerGroups": [], + "isPrimary": false, + "assetDbtJobLastRunStartedAt": 0, + "isDiscoverable": true, + "schemaQualifiedName": "default/snowflake/1666769582/FIVETRAN/DEMO_SALESFORCE", + "viewerUsers": [], + "adminRoles": [], + "adminGroups": [], + "isPinned": false, + "assetDbtJobLastRunCreatedAt": 0, + "databaseQualifiedName": "default/snowflake/1666769582/FIVETRAN", + "assetDbtJobNextRun": 0, + "tableQualifiedName": "default/snowflake/1666769582/FIVETRAN/DEMO_SALESFORCE/DASHBOARD_FEED", + "partitionOrder": 0, + "viewerGroups": [], + "assetDbtJobLastRun": 0 + }, + "guid": "bacab52b-6c4b-4dbe-b5da-97ec2e509c7e", + "status": "ACTIVE", + "displayText": "IS_DELETED", + "isIncomplete": false, + "createdBy": "kartik.thakur", + "updatedBy": "kartik.thakur", + "createTime": 1666792332986, + "updateTime": 1666792332986 + }, + "operationType": "ENTITY_CREATE", + "eventTime": 1666792332986 + } +} diff --git a/tests/unit/data/event_responses/entity_delete.json b/tests/unit/data/event_responses/entity_delete.json new file mode 100644 index 000000000..3cc8b0219 --- /dev/null +++ b/tests/unit/data/event_responses/entity_delete.json @@ -0,0 +1,36 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.121.193.228", + "msgCreatedBy": "", + "msgCreationTime": 1667822854307, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "Connection", + "attributes": { + "qualifiedName": "default/bigquery/1665130698" + }, + "guid": "ff9d053f-5bd8-4149-81d3-447ba654dbaa", + "status": "DELETED", + "displayText": "default/bigquery/1665130698", + "isIncomplete": false, + "createdBy": "autoqa", + "updatedBy": "autoqa", + "createTime": 1665131016428, + "updateTime": 1665131016428, + "deleteHandler": "HARD" + }, + "operationType": "ENTITY_DELETE", + "eventTime": 1667822854152 + } +} diff --git a/tests/unit/data/event_responses/entity_update.json b/tests/unit/data/event_responses/entity_update.json new file mode 100644 index 000000000..0624a1d48 --- /dev/null +++ b/tests/unit/data/event_responses/entity_update.json @@ -0,0 +1,125 @@ +{ + "source": {}, + "version": { + "version": "1.0.0", + "versionParts": [ + 1 + ] + }, + "msgCompressionKind": "NONE", + "msgSplitIdx": 1, + "msgSplitCount": 1, + "msgSourceIP": "10.147.3.150", + "msgCreatedBy": "", + "msgCreationTime": 1705924521864, + "spooled": false, + "message": { + "type": "ENTITY_NOTIFICATION_V2", + "entity": { + "typeName": "AtlasGlossaryTerm", + "attributes": { + "popularityScore": 1.17549435e-38, + "assetMcMonitorNames": [], + "lastSyncRunAt": 0, + "assetSodaLastSyncRunAt": 0, + "starredCount": 0, + "adminUsers": [], + "assetMcIncidentQualifiedNames": [], + "assetMcIncidentTypes": [], + "assetSodaLastScanAt": 0, + "sourceUpdatedAt": 0, + "assetDbtJobLastRunArtifactsSaved": false, + "isEditable": true, + "announcementUpdatedAt": 0, + "sourceCreatedAt": 0, + "assetDbtJobLastRunDequedAt": 0, + "assetDbtTags": [], + "qualifiedName": "8Wi1jGldVz1vEBXhGivg3@79FD59qksQ4G3Y6h5ZWTO", + "assetDbtJobLastRunNotificationsSent": false, + "assetMcMonitorTypes": [], + "assetSodaCheckCount": 0, + "assetMcMonitorStatuses": [], + "starredBy": [], + "name": "new-term", + "certificateUpdatedAt": 1703077797628, + "assetMcIncidentSeverities": [], + "ownerUsers": [ + "pskib" + ], + "certificateStatus": "DRAFT", + "assetDbtJobLastRunHasSourcesGenerated": false, + "assetMcIncidentSubTypes": [], + "isAIGenerated": false, + "assetDbtJobLastRunHasDocsGenerated": false, + "assetTags": [], + "assetMcIncidentStates": [], + "assetDbtJobLastRunUpdatedAt": 0, + "ownerGroups": [], + "certificateUpdatedBy": "pskib", + "assetMcMonitorQualifiedNames": [], + "assetDbtJobLastRunStartedAt": 0, + "isDiscoverable": true, + "isPartial": false, + "assetMcMonitorScheduleTypes": [], + "viewerUsers": [], + "assetMcIncidentNames": [], + "userDescription": "test", + "adminRoles": [], + "adminGroups": [], + "assetDbtJobLastRunCreatedAt": 0, + "assetDbtJobNextRun": 0, + "assetMcLastSyncRunAt": 0, + "viewerGroups": [], + "assetDbtJobLastRun": 0 + }, + "guid": "a5ed097d-93ea-4728-b3c3-ef441c3e6094", + "displayText": "new-term", + "isIncomplete": false, + "createdBy": "pskib", + "updatedBy": "pskib", + "createTime": 1703077797628, + "updateTime": 1705924521736, + "relationshipAttributes": { + "anchor": { + "guid": "579ae112-3f36-40ed-ad58-edcb6e719cf2", + "typeName": "AtlasGlossary", + "attributes": { + "certificateStatus": "DRAFT", + "__modifiedBy": "pskib", + "__state": "ACTIVE", + "__createdBy": "pskib", + "starredBy": [], + "__modificationTimestamp": 1703077797628, + "name": "Test-Glossary", + "isPartial": false, + "assetIcon": "atlanGlossary", + "__timestamp": 1702635066946, + "assetDbtJobLastRun": 0 + }, + "uniqueAttributes": { + "qualifiedName": "79FD59qksQ4G3Y6h5ZWTO" + } + } + } + }, + "operationType": "ENTITY_UPDATE", + "eventTime": 1705924521736, + "mutatedDetails": { + "typeName": "AtlasGlossaryTerm", + "attributes": { + "userDescription": "test" + }, + "guid": "a5ed097d-93ea-4728-b3c3-ef441c3e6094", + "isIncomplete": false, + "provenanceType": 0, + "updatedBy": "pskib", + "updateTime": 1705924521736, + "version": 0, + "proxy": false + }, + "headers": { + "x-atlan-request-id": "e0a11772-8f4b-4141-08e1-4e74998cb0d2", + "x-atlan-via-ui": "true" + } + } +} diff --git a/tests/unit/data/event_responses/validation.json b/tests/unit/data/event_responses/validation.json new file mode 100644 index 000000000..0f235d7b9 --- /dev/null +++ b/tests/unit/data/event_responses/validation.json @@ -0,0 +1,39 @@ +{ + "version": "2.0", + "routeKey": "$default", + "rawPath": "/", + "rawQueryString": "", + "headers": { + "content-length": "85", + "x-amzn-tls-cipher-suite": "ECDHE-RSA-AES128-GCM-SHA256", + "x-amzn-tls-version": "TLSv1.2", + "x-amzn-trace-id": "Root=1-64931f85-02856e0b115997577ed16d97", + "x-forwarded-proto": "https", + "host": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", + "x-forwarded-port": "443", + "content-type": "text/plain; charset=utf-8", + "x-forwarded-for": "34.194.9.164", + "accept-encoding": "gzip", + "user-agent": "go-resty/1.12.0 (https://github.com/go-resty/resty)" + }, + "requestContext": { + "accountId": "anonymous", + "apiId": "uo8rokyifhnhubgagp106xhrpvgmuqhx", + "domainName": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", + "domainPrefix": "uo8rokyifhnhubgagp106xhrpvgmuqhx", + "http": { + "method": "POST", + "path": "/", + "protocol": "HTTP/1.1", + "sourceIp": "34.194.9.164", + "userAgent": "go-resty/1.12.0 (https://github.com/go-resty/resty)" + }, + "requestId": "27539f31-d444-419e-b1ad-4382657b7e04", + "routeKey": "$default", + "stage": "$default", + "time": "21/Jun/2023:16:04:21 +0000", + "timeEpoch": 1687363461544 + }, + "body": "{\"atlan-webhook\": \"Hello, humans of data! It worked. Excited to see what you build!\"}", + "isBase64Encoded": false +} diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index d1dceb252..1301d3a5c 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -1,265 +1,106 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright 2023 Atlan Pte. Ltd. -import json +# Copyright 2024 Atlan Pte. Ltd. +from json import load, loads +from pathlib import Path +import pytest + +from pyatlan.client.atlan import AtlanClient from pyatlan.events.atlan_event_handler import is_validation_request, valid_signature from pyatlan.model.assets import AtlasGlossaryTerm -from pyatlan.model.events import AssetUpdatePayload, AtlanEvent - -SIGNING = "abc123" - -VALIDATION_PAYLOAD = { - "version": "2.0", - "routeKey": "$default", - "rawPath": "/", - "rawQueryString": "", - "headers": { - "content-length": "85", - "x-amzn-tls-cipher-suite": "ECDHE-RSA-AES128-GCM-SHA256", - "x-amzn-tls-version": "TLSv1.2", - "x-amzn-trace-id": "Root=1-64931f85-02856e0b115997577ed16d97", - "x-forwarded-proto": "https", - "host": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", - "x-forwarded-port": "443", - "content-type": "text/plain; charset=utf-8", - "x-forwarded-for": "34.194.9.164", - "accept-encoding": "gzip", - "user-agent": "go-resty/1.12.0 (https://github.com/go-resty/resty)", - }, - "requestContext": { - "accountId": "anonymous", - "apiId": "uo8rokyifhnhubgagp106xhrpvgmuqhx", - "domainName": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", - "domainPrefix": "uo8rokyifhnhubgagp106xhrpvgmuqhx", - "http": { - "method": "POST", - "path": "/", - "protocol": "HTTP/1.1", - "sourceIp": "34.194.9.164", - "userAgent": "go-resty/1.12.0 (https://github.com/go-resty/resty)", - }, - "requestId": "27539f31-d444-419e-b1ad-4382657b7e04", - "routeKey": "$default", - "stage": "$default", - "time": "21/Jun/2023:16:04:21 +0000", - "timeEpoch": 1687363461544, - }, - "body": '{"atlan-webhook": "Hello, humans of data! It worked. Excited to see what you build!"}', - "isBase64Encoded": False, -} - - -ACTUAL_PAYLOAD = { - "version": "2.0", - "routeKey": "$default", - "rawPath": "/", - "rawQueryString": "", - "headers": { - "content-length": "85", - "x-atlan-signing-secret": "abc123", - "x-amzn-tls-cipher-suite": "ECDHE-RSA-AES128-GCM-SHA256", - "x-amzn-tls-version": "TLSv1.2", - "x-amzn-trace-id": "Root=1-64931f85-02856e0b115997577ed16d97", - "x-forwarded-proto": "https", - "host": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", - "x-forwarded-port": "443", - "content-type": "text/plain; charset=utf-8", - "x-forwarded-for": "34.194.9.164", - "accept-encoding": "gzip", - "user-agent": "go-resty/1.12.0 (https://github.com/go-resty/resty)", - }, - "requestContext": { - "accountId": "anonymous", - "apiId": "uo8rokyifhnhubgagp106xhrpvgmuqhx", - "domainName": "uo8rokyifhnhubgagp106xhrpvgmuqhx.lambda-url.us-east-1.on.aws", - "domainPrefix": "uo8rokyifhnhubgagp106xhrpvgmuqhx", - "http": { - "method": "POST", - "path": "/", - "protocol": "HTTP/1.1", - "sourceIp": "34.194.9.164", - "userAgent": "go-resty/1.12.0 (https://github.com/go-resty/resty)", - }, - "requestId": "27539f31-d444-419e-b1ad-4382657b7e04", - "routeKey": "$default", - "stage": "$default", - "time": "21/Jun/2023:16:04:21 +0000", - "timeEpoch": 1687363461544, - }, - "body": '{"source":{},"version":{"version":"1.0.0",' - '"versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":' - '"10.121.193.228","msgCreatedBy":"","msgCreationTime":1666792334952,"spooled":false,"message":{' - '"type":"ENTITY_NOTIFICATION_V2","entity":{"typeName":"AtlasGlossaryTerm","attributes":{' - '"qualifiedName":"rrSsMGHn1q1W2dWHqSHWe@Q07rYkHOACJkmUZJgmKFP","name":"Example"},"guid":' - '"bacab52b-6c4b-4dbe-b5da-97ec2e509c7e","status":"ACTIVE","displayText":"Example"},' - '"operationType":"ENTITY_CREATE","eventTime":1666792332986}}', - "isBase64Encoded": False, -} - -JSON_ATLAN_EVENT = """ -{ - "source": { - }, - "version": { - "version": "1.0.0", - "versionParts": [ - 1 - ] - }, - "msgCompressionKind": "NONE", - "msgSplitIdx": 1, - "msgSplitCount": 1, - "msgSourceIP": "10.147.3.150", - "msgCreatedBy": "", - "msgCreationTime": 1705924521864, - "spooled": false, - "message": { - "type": "ENTITY_NOTIFICATION_V2", - "entity": { - "typeName": "AtlasGlossaryTerm", - "attributes": { - "popularityScore": 1.17549435e-38, - "assetMcMonitorNames": [ - ], - "lastSyncRunAt": 0, - "assetSodaLastSyncRunAt": 0, - "starredCount": 0, - "adminUsers": [ - ], - "assetMcIncidentQualifiedNames": [ - ], - "assetMcIncidentTypes": [ - ], - "assetSodaLastScanAt": 0, - "sourceUpdatedAt": 0, - "assetDbtJobLastRunArtifactsSaved": false, - "isEditable": true, - "announcementUpdatedAt": 0, - "sourceCreatedAt": 0, - "assetDbtJobLastRunDequedAt": 0, - "assetDbtTags": [ - ], - "qualifiedName": "8Wi1jGldVz1vEBXhGivg3@79FD59qksQ4G3Y6h5ZWTO", - "assetDbtJobLastRunNotificationsSent": false, - "assetMcMonitorTypes": [ - ], - "assetSodaCheckCount": 0, - "assetMcMonitorStatuses": [ - ], - "starredBy": [], - "name": "new-term", - "certificateUpdatedAt": 1703077797628, - "assetMcIncidentSeverities": [ - ], - "ownerUsers": [ - "pskib" - ], - "certificateStatus": "DRAFT", - "assetDbtJobLastRunHasSourcesGenerated": false, - "assetMcIncidentSubTypes": [ - ], - "isAIGenerated": false, - "assetDbtJobLastRunHasDocsGenerated": false, - "assetTags": [ - ], - "assetMcIncidentStates": [ - ], - "assetDbtJobLastRunUpdatedAt": 0, - "ownerGroups": [ - ], - "certificateUpdatedBy": "pskib", - "assetMcMonitorQualifiedNames": [ - ], - "assetDbtJobLastRunStartedAt": 0, - "isDiscoverable": true, - "isPartial": false, - "assetMcMonitorScheduleTypes": [ - ], - "viewerUsers": [ - ], - "assetMcIncidentNames": [ - ], - "userDescription": "test", - "adminRoles": [ - ], - "adminGroups": [ - ], - "assetDbtJobLastRunCreatedAt": 0, - "assetDbtJobNextRun": 0, - "assetMcLastSyncRunAt": 0, - "viewerGroups": [ - ], - "assetDbtJobLastRun": 0 - }, - "guid": "a5ed097d-93ea-4728-b3c3-ef441c3e6094", - "displayText": "new-term", - "isIncomplete": false, - "createdBy": "pskib", - "updatedBy": "pskib", - "createTime": 1703077797628, - "updateTime": 1705924521736, - "relationshipAttributes": { - "anchor": { - "guid": "579ae112-3f36-40ed-ad58-edcb6e719cf2", - "typeName": "AtlasGlossary", - "attributes": { - "certificateStatus": "DRAFT", - "__modifiedBy": "pskib", - "__state": "ACTIVE", - "__createdBy": "pskib", - "starredBy": [ - ], - "__modificationTimestamp": 1703077797628, - "name": "Test-Glossary", - "isPartial": false, - "assetIcon": "atlanGlossary", - "__timestamp": 1702635066946, - "assetDbtJobLastRun": 0 - }, - "uniqueAttributes": { - "qualifiedName": "79FD59qksQ4G3Y6h5ZWTO" - } - } - } - }, - "operationType": "ENTITY_UPDATE", - "eventTime": 1705924521736, - "mutatedDetails": { - "typeName": "AtlasGlossaryTerm", - "attributes": { - "userDescription": "test" - }, - "guid": "a5ed097d-93ea-4728-b3c3-ef441c3e6094", - "isIncomplete": false, - "provenanceType": 0, - "updatedBy": "pskib", - "updateTime": 1705924521736, - "version": 0, - "proxy": false - }, - "headers": { - "x-atlan-request-id": "e0a11772-8f4b-4141-08e1-4e74998cb0d2", - "x-atlan-via-ui": "true" - } - } -}""" +from pyatlan.model.events import ( + AssetCreatePayload, + AssetDeletePayload, + AssetUpdatePayload, + AtlanEvent, + AtlanTagAddPayload, + AtlanTagDeletePayload, + CustomMetadataUpdatePayload, +) + +ACTUAL_JSON = "actual.json" +VALIDATION_JSON = "validation.json" +ENTITY_CREATE_JSON = "entity_create.json" +ENTITY_UPDATE_JSON = "entity_update.json" +ENTITY_DELETE_JSON = "entity_delete.json" +CLASSIFICATION_ADD_JSON = "classification_add.json" +CLASSIFICATION_DELETE_JSON = "classification_delete.json" +BUSINESS_ATTRIBUTE_UPDATE_JSON = "business_attribute_update.json" +TEST_DATA_DIR = Path(__file__).parent / "data" +EVENT_RESPONSES_DIR = TEST_DATA_DIR / "event_responses" + + +def load_json(respones_dir, filename): + with (respones_dir / filename).open() as input_file: + return load(input_file) + + +@pytest.fixture(autouse=True) +def set_env(monkeypatch): + monkeypatch.setenv("ATLAN_API_KEY", "test-api-key") + monkeypatch.setenv("ATLAN_BASE_URL", "https://test.atlan.com") + + +@pytest.fixture() +def client(): + return AtlanClient() + + +@pytest.fixture() +def actual_json(): + return load_json(EVENT_RESPONSES_DIR, ACTUAL_JSON) + + +@pytest.fixture() +def validation_json(): + return load_json(EVENT_RESPONSES_DIR, VALIDATION_JSON) + +@pytest.fixture() +def entity_create_json(): + return load_json(EVENT_RESPONSES_DIR, ENTITY_CREATE_JSON) -def test_validation_payload(): - body = VALIDATION_PAYLOAD.get("body") + +@pytest.fixture() +def entity_update_json(): + return load_json(EVENT_RESPONSES_DIR, ENTITY_UPDATE_JSON) + + +@pytest.fixture() +def entity_delete_json(): + return load_json(EVENT_RESPONSES_DIR, ENTITY_DELETE_JSON) + + +@pytest.fixture() +def custom_metadata_add_json(): + return load_json(EVENT_RESPONSES_DIR, BUSINESS_ATTRIBUTE_UPDATE_JSON) + + +@pytest.fixture() +def tag_add_json(): + return load_json(EVENT_RESPONSES_DIR, CLASSIFICATION_ADD_JSON) + + +@pytest.fixture() +def tag_delete_json(): + return load_json(EVENT_RESPONSES_DIR, CLASSIFICATION_DELETE_JSON) + + +def test_validation_payload(validation_json): + body = validation_json.get("body") assert is_validation_request(body) -def test_no_signing_key(): - assert not valid_signature(SIGNING, VALIDATION_PAYLOAD.get("headers")) +def test_no_signing_key(validation_json): + assert not valid_signature("test-secret", validation_json.get("headers")) -def test_signing_key(): - assert valid_signature(SIGNING, ACTUAL_PAYLOAD.get("headers")) +def test_signing_key(actual_json): + assert valid_signature("test-secret", actual_json.get("headers")) -def test_body(): - body = json.loads(ACTUAL_PAYLOAD.get("body")) +def test_body(actual_json): + body = loads(actual_json.get("body")) assert body atlan_event = AtlanEvent(**body) assert atlan_event @@ -267,7 +108,26 @@ def test_body(): assert isinstance(atlan_event.payload.asset, AtlasGlossaryTerm) -def test_correct_payload_type_returned(): - payload = json.loads(JSON_ATLAN_EVENT) - event = AtlanEvent(**payload) - assert isinstance(event.payload, AssetUpdatePayload) +def test_atlan_events_deserialization( + client, + mock_tag_cache, + entity_create_json, + entity_update_json, + entity_delete_json, + tag_add_json, + tag_delete_json, + custom_metadata_add_json, +): + _EVENT_TYPES = { + "entity_create_json": AssetCreatePayload, + "entity_update_json": AssetUpdatePayload, + "entity_delete_json": AssetDeletePayload, + "tag_add_json": AtlanTagAddPayload, + "tag_delete_json": AtlanTagDeletePayload, + "custom_metadata_add_json": CustomMetadataUpdatePayload, + } + + for key, payload_type in _EVENT_TYPES.items(): + data = locals()[key] + event = AtlanEvent(**data) + assert isinstance(event.payload, payload_type) From f7264c31f4c440fa770a7a6b11ea7ee96738d54d Mon Sep 17 00:00:00 2001 From: Aryamanz29 Date: Wed, 3 Apr 2024 17:30:02 +0530 Subject: [PATCH 2/2] Fixes field alias typo for `restrict_propagation_through_hierarchy` in the `AtlanTag` model --- pyatlan/model/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyatlan/model/core.py b/pyatlan/model/core.py index f68e94cf8..426f401f3 100644 --- a/pyatlan/model/core.py +++ b/pyatlan/model/core.py @@ -180,7 +180,7 @@ class Config: "Whether to prevent this Atlan tag from propagating through " "hierarchy (True) or allow it to propagate through hierarchy (False)" ), - alias="restrictPropagationThroughHierachy", + alias="restrictPropagationThroughHierarchy", ) validity_periods: Optional[List[str]] = Field(default=None, alias="validityPeriods") _source_tag_attachements: List[SourceTagAttachment] = PrivateAttr(