From e0242c8ebbc472371555a13bfb28e92ab4a181ec Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Thu, 13 Feb 2025 17:03:57 -0500 Subject: [PATCH 01/12] source-zendesk-support-native: update TIME_PARAMETER_DELAY to avoid intermittent snapshot test failures When testing locally, I'd intermittently get snapshot test failures because a start_time param was too recent. It turns out that Zendesk returns that error when the start_time is 60 or fewer seconds from the present. Bumping the delay up to 61 seconds avoids these errors. --- .../source_zendesk_support_native/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index 9bb38bbd1a..ea68b05838 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -26,8 +26,8 @@ CURSOR_PAGINATION_PAGE_SIZE = 100 MAX_SATISFACTION_RATINGS_WINDOW_SIZE = timedelta(days=30) -# Zendesk errors out if a start or end time parameter is more recent than 60 seconds in the past. -TIME_PARAMETER_DELAY = timedelta(seconds=60) +# Zendesk errors out if a start or end time parameter is 60 seconds or less in the past. +TIME_PARAMETER_DELAY = timedelta(seconds=61) DATETIME_STRING_FORMAT = "%Y-%m-%dT%H:%M:%SZ" From 77cd0023127fb11103146bea75793e7d05a5d2ba Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Thu, 13 Feb 2025 17:31:22 -0500 Subject: [PATCH 02/12] source-zendesk-support-native: add `sla_policies` stream --- .../acmeCo/flow.yaml | 4 ++ .../acmeCo/sla_policies.schema.yaml | 31 ++++++++ .../source_zendesk_support_native/api.py | 31 ++++++++ .../source_zendesk_support_native/models.py | 21 +++++- .../resources.py | 50 +++++++++++++ source-zendesk-support-native/test.flow.yaml | 4 ++ .../snapshots__capture__capture.stdout.json | 72 +++++++++++++++++++ .../snapshots__discover__capture.stdout.json | 51 +++++++++++++ 8 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 source-zendesk-support-native/acmeCo/sla_policies.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index 7691c9fb20..276c034817 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -24,6 +24,10 @@ collections: schema: satisfaction_ratings.schema.yaml key: - /id + acmeCo/sla_policies: + schema: sla_policies.schema.yaml + key: + - /_meta/row_id acmeCo/tags: schema: tags.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/sla_policies.schema.yaml b/source-zendesk-support-native/acmeCo/sla_policies.schema.yaml new file mode 100644 index 0000000000..f5a89babde --- /dev/null +++ b/source-zendesk-support-native/acmeCo/sla_policies.schema.yaml @@ -0,0 +1,31 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata +title: FullRefreshResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index ea68b05838..c5b153399c 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -9,6 +9,7 @@ from .models import ( FullRefreshResource, + FullRefreshOffsetPaginatedResponse, FullRefreshCursorPaginatedResponse, ZendeskResource, TimestampedResource, @@ -67,6 +68,36 @@ def _is_timestamp(string: str) -> bool: return False +async def snapshot_offset_paginated_resources( + http: HTTPSession, + subdomain: str, + path: str, + response_model: type[FullRefreshOffsetPaginatedResponse], + log: Logger, +) -> AsyncGenerator[FullRefreshResource, None]: + url = f"{url_base(subdomain)}/{path}" + page_num = 1 + params: dict[str, str | int] = { + "per_page": CURSOR_PAGINATION_PAGE_SIZE, + "page": page_num, + } + + while True: + response = response_model.model_validate_json( + await http.request(log, url, params=params) + ) + + for resource in response.resources: + yield resource + + if not response.next_page: + return + + page_num += 1 + + params["page"] = page_num + + async def snapshot_cursor_paginated_resources( http: HTTPSession, subdomain: str, diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 31d0b997eb..8a16982318 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -152,7 +152,26 @@ class UsersResponse(IncrementalCursorExportResponse): ] -class FullRefreshCursorPaginatedResponse(BaseModel, extra="allow"): +class FullRefreshResponse(BaseModel, extra="allow"): + resources: list[FullRefreshResource] + + +class FullRefreshOffsetPaginatedResponse(FullRefreshResponse): + next_page: str | None + + +class SlaPoliciesResponse(FullRefreshOffsetPaginatedResponse): + resources: list[FullRefreshResource] = Field(alias="sla_policies") + + +# Full refresh resources that paginted with page offsets. +# Tuples contain the name, path, and response model for each resource. +FULL_REFRESH_OFFSET_PAGINATED_RESOURCES: list[tuple[str, str, type[FullRefreshOffsetPaginatedResponse]]] = [ + ("sla_policies", "slas/policies", SlaPoliciesResponse), +] + + +class FullRefreshCursorPaginatedResponse(FullRefreshResponse): class Meta(BaseModel, extra="forbid"): has_more: bool after_cursor: str | None diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index efd266bfab..974bb5a460 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -11,6 +11,7 @@ AuditLog, ClientSideIncrementalCursorPaginatedResponse, EndpointConfig, + FullRefreshOffsetPaginatedResponse, FullRefreshCursorPaginatedResponse, FullRefreshResource, IncrementalCursorPaginatedResponse, @@ -20,6 +21,7 @@ ZendeskResource, EPOCH, CLIENT_SIDE_FILTERED_CURSOR_PAGINATED_RESOURCES, + FULL_REFRESH_OFFSET_PAGINATED_RESOURCES, FULL_REFRESH_CURSOR_PAGINATED_RESOURCES, INCREMENTAL_CURSOR_EXPORT_RESOURCES, INCREMENTAL_CURSOR_EXPORT_TYPES, @@ -41,6 +43,7 @@ fetch_satisfaction_ratings, fetch_ticket_child_resources, fetch_ticket_metrics, + snapshot_offset_paginated_resources, snapshot_cursor_paginated_resources, url_base, _dt_to_s, @@ -164,6 +167,52 @@ def open( ) +def full_refresh_offset_paginated_resources( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> list[common.Resource]: + + def open( + path: str, + response_model: type[FullRefreshOffsetPaginatedResponse], + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_snapshot=functools.partial( + snapshot_offset_paginated_resources, + http, + config.subdomain, + path, + response_model, + ), + tombstone=FullRefreshResource(_meta=FullRefreshResource.Meta(op="d")) + ) + + resources = [ + common.Resource( + name=name, + key=["/_meta/row_id"], + model=FullRefreshResource, + open=functools.partial(open, path, response_model), + initial_state=ResourceState(), + initial_config=ResourceConfig( + name=name, interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + for (name, path, response_model) in FULL_REFRESH_OFFSET_PAGINATED_RESOURCES + ] + + return resources + + def full_refresh_cursor_paginated_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -499,6 +548,7 @@ async def all_resources( return [ audit_logs(log, http, config), ticket_metrics(log, http, config), + *full_refresh_offset_paginated_resources(log, http, config), *full_refresh_cursor_paginated_resources(log, http, config), *client_side_filtered_cursor_paginated_resources(log, http, config), satisfaction_ratings(log, http, config), diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index a8efb7d9d3..deffc2ce49 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -19,6 +19,10 @@ captures: name: ticket_metrics interval: PT5M target: acmeCo/ticket_metrics + - resource: + name: sla_policies + interval: PT5M + target: acmeCo/sla_policies - resource: name: tags interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index fafdbd5f4e..aea00c342d 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -113,6 +113,78 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/satisfaction_ratings/29013431981588.json" } ], + [ + "acmeCo/sla_policies", + { + "_meta": { + "op": "c", + "row_id": 0, + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef" + }, + "created_at": "2024-08-01T13:54:40Z", + "description": "Details copied form Zendesk API code sample. For urgent incidents, we will respond to tickets in 10 minutes", + "filter": { + "all": [ + { + "field": "ticket_type_id", + "operator": "is", + "value": "2" + } + ], + "any": [] + }, + "id": 29006807867284, + "metric_settings": null, + "policy_metrics": [ + { + "business_hours": false, + "metric": "requester_wait_time", + "priority": "low", + "target": 180, + "target_in_seconds": 10800 + }, + { + "business_hours": false, + "metric": "first_reply_time", + "priority": "normal", + "target": 30, + "target_in_seconds": 1800 + }, + { + "business_hours": false, + "metric": "requester_wait_time", + "priority": "normal", + "target": 160, + "target_in_seconds": 9600 + }, + { + "business_hours": false, + "metric": "requester_wait_time", + "priority": "high", + "target": 140, + "target_in_seconds": 8400 + }, + { + "business_hours": false, + "metric": "first_reply_time", + "priority": "urgent", + "target": 10, + "target_in_seconds": 600 + }, + { + "business_hours": false, + "metric": "requester_wait_time", + "priority": "urgent", + "target": 120, + "target_in_seconds": 7200 + } + ], + "position": 1, + "title": "Test SLA Policy", + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/slas/policies/29006807867284.json" + } + ], [ "acmeCo/ticket_audits", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 634a9080a3..a866e89fdd 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -377,6 +377,57 @@ "/id" ] }, + { + "recommendedName": "sla_policies", + "resourceConfig": { + "name": "sla_policies", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + } + }, + "title": "FullRefreshResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, { "recommendedName": "tags", "resourceConfig": { From 41bf4752f955fd0f04c8025dfa1c85d7bab2dac6 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Thu, 13 Feb 2025 17:47:16 -0500 Subject: [PATCH 03/12] source-zendesk-support-native: add `group_memberships` stream --- .../acmeCo/flow.yaml | 4 ++ .../acmeCo/group_memberships.schema.yaml | 41 ++++++++++++ .../source_zendesk_support_native/models.py | 5 ++ source-zendesk-support-native/test.flow.yaml | 4 ++ .../snapshots__capture__capture.stdout.json | 16 +++++ .../snapshots__discover__capture.stdout.json | 64 +++++++++++++++++++ 6 files changed, 134 insertions(+) create mode 100644 source-zendesk-support-native/acmeCo/group_memberships.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index 276c034817..a8bded746c 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -8,6 +8,10 @@ collections: schema: brands.schema.yaml key: - /id + acmeCo/group_memberships: + schema: group_memberships.schema.yaml + key: + - /id acmeCo/groups: schema: groups.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/group_memberships.schema.yaml b/source-zendesk-support-native/acmeCo/group_memberships.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/group_memberships.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 8a16982318..087b473ce0 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -253,6 +253,10 @@ class GroupsResponse(ClientSideIncrementalCursorPaginatedResponse): resources: list[TimestampedResource] = Field(alias="groups") +class GroupMembershipsResponse(ClientSideIncrementalCursorPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="group_memberships") + + class MacrosResponse(ClientSideIncrementalCursorPaginatedResponse): resources: list[TimestampedResource] = Field(alias="macros") @@ -266,6 +270,7 @@ class OrganizationMembershipsResponse(ClientSideIncrementalCursorPaginatedRespon CLIENT_SIDE_FILTERED_CURSOR_PAGINATED_RESOURCES: list[tuple[str, str, dict[str, str | int] | None, type[ClientSideIncrementalCursorPaginatedResponse]]] = [ ("brands", "brands", None, BrandsResponse), ("groups", "groups", {"exclude_deleted": "false"}, GroupsResponse), + ("group_memberships", "group_memberships", None, GroupMembershipsResponse), ("macros", "macros", None, MacrosResponse), ("organization_memberships", "organization_memberships", None, OrganizationMembershipsResponse), ("ticket_fields", "ticket_fields", None, TicketFieldsResponse), diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index deffc2ce49..b8d26ec11a 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -35,6 +35,10 @@ captures: name: groups interval: PT5M target: acmeCo/groups + - resource: + name: group_memberships + interval: PT5M + target: acmeCo/group_memberships - resource: name: macros interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index aea00c342d..1e6059989c 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -26,6 +26,22 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/brands/28835714705684.json" } ], + [ + "acmeCo/group_memberships", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "created_at": "2024-07-26T15:41:00Z", + "default": true, + "group_id": 28835740822676, + "id": 28835714743188, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/group_memberships/28835714743188.json", + "user_id": 28835702984212 + } + ], [ "acmeCo/groups", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index a866e89fdd..b0f9fc76b2 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -127,6 +127,70 @@ "/id" ] }, + { + "recommendedName": "group_memberships", + "resourceConfig": { + "name": "group_memberships", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "groups", "resourceConfig": { From 7189c5c26df737118e3025737edceb78df9d2b9d Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Thu, 13 Feb 2025 17:57:44 -0500 Subject: [PATCH 04/12] source-zendesk-support-native: add `custom_roles` and `ticket_forms` streams --- .../acmeCo/custom_roles.schema.yaml | 41 ++++++ .../acmeCo/flow.yaml | 8 ++ .../acmeCo/ticket_forms.schema.yaml | 41 ++++++ .../source_zendesk_support_native/api.py | 42 ++++++ .../source_zendesk_support_native/models.py | 20 +++ .../resources.py | 53 ++++++++ source-zendesk-support-native/test.flow.yaml | 8 ++ .../snapshots__capture__capture.stdout.json | 112 +++++++++++++++ .../snapshots__discover__capture.stdout.json | 128 ++++++++++++++++++ 9 files changed, 453 insertions(+) create mode 100644 source-zendesk-support-native/acmeCo/custom_roles.schema.yaml create mode 100644 source-zendesk-support-native/acmeCo/ticket_forms.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/custom_roles.schema.yaml b/source-zendesk-support-native/acmeCo/custom_roles.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/custom_roles.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index a8bded746c..39f6d912fb 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -8,6 +8,10 @@ collections: schema: brands.schema.yaml key: - /id + acmeCo/custom_roles: + schema: custom_roles.schema.yaml + key: + - /id acmeCo/group_memberships: schema: group_memberships.schema.yaml key: @@ -48,6 +52,10 @@ collections: schema: ticket_fields.schema.yaml key: - /id + acmeCo/ticket_forms: + schema: ticket_forms.schema.yaml + key: + - /id acmeCo/ticket_metric_events: schema: ticket_metric_events.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/ticket_forms.schema.yaml b/source-zendesk-support-native/acmeCo/ticket_forms.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/ticket_forms.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index c5b153399c..8d02a3c237 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -17,6 +17,7 @@ IncrementalCursorExportResponse, TicketsResponse, UsersResponse, + ClientSideIncrementalOffsetPaginatedResponse, ClientSideIncrementalCursorPaginatedResponse, IncrementalCursorPaginatedResponse, SatisfactionRatingsResponse, @@ -125,6 +126,47 @@ async def snapshot_cursor_paginated_resources( params["page[after]"] = response.meta.after_cursor +async def fetch_client_side_incremental_offset_paginated_resources( + http: HTTPSession, + subdomain: str, + path: str, + response_model: type[ClientSideIncrementalOffsetPaginatedResponse], + log: Logger, + log_cursor: LogCursor, +) -> AsyncGenerator[TimestampedResource | LogCursor, None]: + assert isinstance(log_cursor, datetime) + + url = f"{url_base(subdomain)}/{path}" + page_num = 1 + params: dict[str, str | int] = { + "per_page": CURSOR_PAGINATION_PAGE_SIZE, + "page": page_num, + } + + last_seen = log_cursor + + while True: + response = response_model.model_validate_json( + await http.request(log, url, params=params) + ) + + for resource in response.resources: + if resource.updated_at > log_cursor: + yield resource + + if resource.updated_at > last_seen: + last_seen = resource.updated_at + + if not response.next_page: + break + + page_num += 1 + params["page"] = page_num + + if last_seen > log_cursor: + yield last_seen + + async def fetch_client_side_incremental_cursor_paginated_resources( http: HTTPSession, subdomain: str, diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 087b473ce0..5a7ecc94e7 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -237,6 +237,26 @@ class AbbreviatedTicket(BaseModel): ] +class ClientSideIncrementalOffsetPaginatedResponse(FullRefreshOffsetPaginatedResponse): + resources: list[TimestampedResource] + + +class CustomRolesResponse(ClientSideIncrementalOffsetPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="custom_roles") + + +class TicketFormsResponse(ClientSideIncrementalOffsetPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="ticket_forms") + + +# Incremental client side resources that are paginated with page offsets. +# Tuples contain the name, path, and response model for each resource. +CLIENT_SIDE_FILTERED_OFFSET_PAGINATED_RESOURCES: list[tuple[str, str, type[ClientSideIncrementalOffsetPaginatedResponse]]] = [ + ("custom_roles", "custom_roles", CustomRolesResponse), + ("ticket_forms", "ticket_forms", TicketFormsResponse), +] + + class ClientSideIncrementalCursorPaginatedResponse(FullRefreshCursorPaginatedResponse): resources: list[TimestampedResource] diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index 974bb5a460..0b8f45a1dc 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -9,6 +9,7 @@ from .models import ( AuditLog, + ClientSideIncrementalOffsetPaginatedResponse, ClientSideIncrementalCursorPaginatedResponse, EndpointConfig, FullRefreshOffsetPaginatedResponse, @@ -20,6 +21,7 @@ TimestampedResource, ZendeskResource, EPOCH, + CLIENT_SIDE_FILTERED_OFFSET_PAGINATED_RESOURCES, CLIENT_SIDE_FILTERED_CURSOR_PAGINATED_RESOURCES, FULL_REFRESH_OFFSET_PAGINATED_RESOURCES, FULL_REFRESH_CURSOR_PAGINATED_RESOURCES, @@ -37,6 +39,7 @@ backfill_ticket_child_resources, backfill_ticket_metrics, fetch_audit_logs, + fetch_client_side_incremental_offset_paginated_resources, fetch_client_side_incremental_cursor_paginated_resources, fetch_incremental_cursor_export_resources, fetch_incremental_cursor_paginated_resources, @@ -259,6 +262,55 @@ def open( return resources +def client_side_filtered_offset_paginated_resources( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> list[common.Resource]: + + def open( + path: str, + response_model: type[ClientSideIncrementalOffsetPaginatedResponse], + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_changes=functools.partial( + fetch_client_side_incremental_offset_paginated_resources, + http, + config.subdomain, + path, + response_model, + ), + ) + + resources = [ + common.Resource( + name=name, + key=["/id"], + model=TimestampedResource, + open=functools.partial(open, path, response_model), + initial_state=ResourceState( + # Set the initial state of these streams to be the epoch so all results are initially + # emitted, then only updated results are emitted on subsequent sweeps. + inc=ResourceState.Incremental(cursor=EPOCH) + ), + initial_config=ResourceConfig( + name=name, interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + for (name, path, response_model) in CLIENT_SIDE_FILTERED_OFFSET_PAGINATED_RESOURCES + ] + + return resources + + def client_side_filtered_cursor_paginated_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -550,6 +602,7 @@ async def all_resources( ticket_metrics(log, http, config), *full_refresh_offset_paginated_resources(log, http, config), *full_refresh_cursor_paginated_resources(log, http, config), + *client_side_filtered_offset_paginated_resources(log, http, config), *client_side_filtered_cursor_paginated_resources(log, http, config), satisfaction_ratings(log, http, config), *incremental_cursor_paginated_resources(log, http, config), diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index b8d26ec11a..f12c3fd152 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -27,6 +27,14 @@ captures: name: tags interval: PT5M target: acmeCo/tags + - resource: + name: custom_roles + interval: PT5M + target: acmeCo/custom_roles + - resource: + name: ticket_forms + interval: PT5M + target: acmeCo/ticket_forms - resource: name: brands interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index 1e6059989c..d890103c10 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -26,6 +26,82 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/brands/28835714705684.json" } ], + [ + "acmeCo/custom_roles", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "configuration": { + "assign_agent_statuses": false, + "assign_tickets_to_any_group": false, + "chat_access": false, + "custom_objects": {}, + "end_user_list_access": "full", + "end_user_profile_access": "readonly", + "explore_access": "readonly", + "explore_reports": { + "ticket_access": "all", + "ticket_access_selected_groups": [] + }, + "forum_access": "readonly", + "forum_access_restricted_content": false, + "group_access": false, + "light_agent": true, + "macro_access": "readonly", + "manage_automations": false, + "manage_business_rules": false, + "manage_contextual_workspaces": false, + "manage_deletion_schedules": "none", + "manage_dynamic_content": false, + "manage_extensions_and_channels": false, + "manage_facebook": false, + "manage_group_memberships": false, + "manage_groups": false, + "manage_macro_content_suggestions": false, + "manage_malicious_attachments": false, + "manage_organization_fields": false, + "manage_organizations": false, + "manage_roles": "none", + "manage_skills": false, + "manage_slas": false, + "manage_suspended_tickets": false, + "manage_ticket_fields": false, + "manage_ticket_forms": false, + "manage_triggers": false, + "manage_user_fields": false, + "moderate_forums": false, + "organization_editing": false, + "organization_notes_editing": false, + "read_macro_content_suggestions": false, + "report_access": "readonly", + "side_conversation_create": true, + "ticket_access": "within-groups", + "ticket_comment_access": "none", + "ticket_deletion": false, + "ticket_editing": false, + "ticket_merge": false, + "ticket_redaction": false, + "ticket_tag_editing": false, + "twitter_search_access": false, + "user_view_access": "none", + "view_access": "readonly", + "view_deleted_tickets": false, + "view_filter_tickets": true, + "view_reduced_count": false, + "voice_access": false, + "voice_dashboard_access": false + }, + "created_at": "2024-07-26T15:40:56Z", + "description": "Can view and add private comments to tickets", + "id": 28835714382484, + "name": "Light agent", + "role_type": 1, + "team_member_count": 0, + "updated_at": "redacted" + } + ], [ "acmeCo/group_memberships", { @@ -389,6 +465,42 @@ "visible_in_portal": true } ], + [ + "acmeCo/ticket_forms", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "active": true, + "agent_conditions": [], + "created_at": "2024-07-26T15:40:58Z", + "default": true, + "display_name": "Default Ticket Form", + "end_user_conditions": [], + "end_user_visible": true, + "id": 28835740693908, + "in_all_brands": true, + "name": "Default Ticket Form", + "position": 0, + "raw_display_name": "Default Ticket Form", + "raw_name": "Default Ticket Form", + "restricted_brand_ids": [], + "ticket_field_ids": [ + 28835714573076, + 28835740681620, + 28835740683540, + 28835740685844, + 28835714576020, + 28835740687508, + 28835740688660, + 28835714575124, + 28835767646868 + ], + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/ticket_forms/28835740693908.json" + } + ], [ "acmeCo/ticket_metric_events", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index b0f9fc76b2..3d808493ed 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -127,6 +127,70 @@ "/id" ] }, + { + "recommendedName": "custom_roles", + "resourceConfig": { + "name": "custom_roles", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "group_memberships", "resourceConfig": { @@ -723,6 +787,70 @@ "/id" ] }, + { + "recommendedName": "ticket_forms", + "resourceConfig": { + "name": "ticket_forms", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "ticket_metric_events", "resourceConfig": { From cc8e0443b056476e281b4e33d7cb2b0855ab42ee Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Thu, 13 Feb 2025 20:49:21 -0500 Subject: [PATCH 05/12] source-zendesk-support-native: add `schedules` stream --- .../acmeCo/flow.yaml | 4 ++ .../acmeCo/schedules.schema.yaml | 31 +++++++++++ .../source_zendesk_support_native/api.py | 18 +++++++ .../source_zendesk_support_native/models.py | 11 ++++ .../resources.py | 50 ++++++++++++++++++ source-zendesk-support-native/test.flow.yaml | 4 ++ .../snapshots__capture__capture.stdout.json | 37 ++++++++++++++ .../snapshots__discover__capture.stdout.json | 51 +++++++++++++++++++ 8 files changed, 206 insertions(+) create mode 100644 source-zendesk-support-native/acmeCo/schedules.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index 39f6d912fb..3d6a4287c9 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -32,6 +32,10 @@ collections: schema: satisfaction_ratings.schema.yaml key: - /id + acmeCo/schedules: + schema: schedules.schema.yaml + key: + - /_meta/row_id acmeCo/sla_policies: schema: sla_policies.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/schedules.schema.yaml b/source-zendesk-support-native/acmeCo/schedules.schema.yaml new file mode 100644 index 0000000000..f5a89babde --- /dev/null +++ b/source-zendesk-support-native/acmeCo/schedules.schema.yaml @@ -0,0 +1,31 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata +title: FullRefreshResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index 8d02a3c237..8f076ed770 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -9,6 +9,7 @@ from .models import ( FullRefreshResource, + FullRefreshResponse, FullRefreshOffsetPaginatedResponse, FullRefreshCursorPaginatedResponse, ZendeskResource, @@ -69,6 +70,23 @@ def _is_timestamp(string: str) -> bool: return False +async def snapshot_resources( + http: HTTPSession, + subdomain: str, + path: str, + response_model: type[FullRefreshResponse], + log: Logger, +) -> AsyncGenerator[FullRefreshResource, None]: + url = f"{url_base(subdomain)}/{path}" + + response = response_model.model_validate_json( + await http.request(log, url) + ) + + for resource in response.resources: + yield resource + + async def snapshot_offset_paginated_resources( http: HTTPSession, subdomain: str, diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 5a7ecc94e7..b4d2b3c3f3 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -156,6 +156,17 @@ class FullRefreshResponse(BaseModel, extra="allow"): resources: list[FullRefreshResource] +class SchedulesResponse(FullRefreshResponse): + resources: list[FullRefreshResource] = Field(alias="schedules") + + +# Full refresh resources with no pagination. +# Tuples contain the name, path, and response model for each resource. +FULL_REFRESH_RESOURCES: list[tuple[str, str, type[FullRefreshResponse]]] = [ + ("schedules", "business_hours/schedules", SchedulesResponse), +] + + class FullRefreshOffsetPaginatedResponse(FullRefreshResponse): next_page: str | None diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index 0b8f45a1dc..4266981cf1 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -12,6 +12,7 @@ ClientSideIncrementalOffsetPaginatedResponse, ClientSideIncrementalCursorPaginatedResponse, EndpointConfig, + FullRefreshResponse, FullRefreshOffsetPaginatedResponse, FullRefreshCursorPaginatedResponse, FullRefreshResource, @@ -23,6 +24,7 @@ EPOCH, CLIENT_SIDE_FILTERED_OFFSET_PAGINATED_RESOURCES, CLIENT_SIDE_FILTERED_CURSOR_PAGINATED_RESOURCES, + FULL_REFRESH_RESOURCES, FULL_REFRESH_OFFSET_PAGINATED_RESOURCES, FULL_REFRESH_CURSOR_PAGINATED_RESOURCES, INCREMENTAL_CURSOR_EXPORT_RESOURCES, @@ -46,6 +48,7 @@ fetch_satisfaction_ratings, fetch_ticket_child_resources, fetch_ticket_metrics, + snapshot_resources, snapshot_offset_paginated_resources, snapshot_cursor_paginated_resources, url_base, @@ -170,6 +173,52 @@ def open( ) +def full_refresh_resources( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> list[common.Resource]: + + def open( + path: str, + response_model: type[FullRefreshResponse], + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_snapshot=functools.partial( + snapshot_resources, + http, + config.subdomain, + path, + response_model, + ), + tombstone=FullRefreshResource(_meta=FullRefreshResource.Meta(op="d")) + ) + + resources = [ + common.Resource( + name=name, + key=["/_meta/row_id"], + model=FullRefreshResource, + open=functools.partial(open, path, response_model), + initial_state=ResourceState(), + initial_config=ResourceConfig( + name=name, interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + for (name, path, response_model) in FULL_REFRESH_RESOURCES + ] + + return resources + + def full_refresh_offset_paginated_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -600,6 +649,7 @@ async def all_resources( return [ audit_logs(log, http, config), ticket_metrics(log, http, config), + *full_refresh_resources(log, http, config), *full_refresh_offset_paginated_resources(log, http, config), *full_refresh_cursor_paginated_resources(log, http, config), *client_side_filtered_offset_paginated_resources(log, http, config), diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index f12c3fd152..9fc725f692 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -19,6 +19,10 @@ captures: name: ticket_metrics interval: PT5M target: acmeCo/ticket_metrics + - resource: + name: schedules + interval: PT5M + target: acmeCo/schedules - resource: name: sla_policies interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index d890103c10..1e4f9242d8 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -205,6 +205,43 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/satisfaction_ratings/29013431981588.json" } ], + [ + "acmeCo/schedules", + { + "_meta": { + "op": "c", + "row_id": 0, + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef" + }, + "created_at": "2024-08-01T13:29:59Z", + "id": 29006094707860, + "intervals": [ + { + "end_time": 2460, + "start_time": 1980 + }, + { + "end_time": 3900, + "start_time": 3420 + }, + { + "end_time": 5340, + "start_time": 4860 + }, + { + "end_time": 6780, + "start_time": 6300 + }, + { + "end_time": 8220, + "start_time": 7740 + } + ], + "name": "Test schedule", + "time_zone": "Eastern Time (US & Canada)", + "updated_at": "redacted" + } + ], [ "acmeCo/sla_policies", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 3d808493ed..0b0d1c1294 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -505,6 +505,57 @@ "/id" ] }, + { + "recommendedName": "schedules", + "resourceConfig": { + "name": "schedules", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + } + }, + "title": "FullRefreshResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, { "recommendedName": "sla_policies", "resourceConfig": { From 2041cf758545e1dc51eeba12aaa7ca4a95a72a9b Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 10:18:18 -0500 Subject: [PATCH 06/12] source-zendesk-support-native: add `organizations` stream The `organizations` stream uses the incremental time based export endpoint, which has the following quirks: - A 10 requests / minute rate limit - The next page is fetched using the updated_at timestamp of the last result in the current response. This means the next page will always contain at least one duplicate record (the last record of the previous page). Because of these quirks: - A super basic rate limiting strategy leveraging asyncio.Lock is used to only make 1 request every 6 seconds to incremental time export endpoints. - An error is raised if we detect 1,000+ organizations updated at the same time. If we didn't the connector would be stuck looping & making the same request endlessly since the max page size is 1,000. - Checkpoints can't be emitted at page breaks since the next page's first result overlaps with the previous page's end, and it's possible to miss records between connector restarts with that strategy. --- .../acmeCo/flow.yaml | 4 + .../acmeCo/organizations.schema.yaml | 41 ++++++ .../source_zendesk_support_native/api.py | 125 +++++++++++++++++- .../source_zendesk_support_native/models.py | 18 +++ .../resources.py | 65 +++++++++ source-zendesk-support-native/test.flow.yaml | 4 + .../snapshots__capture__capture.stdout.json | 26 ++++ .../snapshots__discover__capture.stdout.json | 64 +++++++++ 8 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 source-zendesk-support-native/acmeCo/organizations.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index 3d6a4287c9..ea2e891242 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -28,6 +28,10 @@ collections: schema: organization_memberships.schema.yaml key: - /id + acmeCo/organizations: + schema: organizations.schema.yaml + key: + - /id acmeCo/satisfaction_ratings: schema: satisfaction_ratings.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/organizations.schema.yaml b/source-zendesk-support-native/acmeCo/organizations.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/organizations.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index 8f076ed770..14e472f9f8 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -1,3 +1,4 @@ +import asyncio import base64 from datetime import datetime, timedelta, UTC from logging import Logger @@ -15,7 +16,7 @@ ZendeskResource, TimestampedResource, AbbreviatedTicket, - IncrementalCursorExportResponse, + IncrementalTimeExportResponse, TicketsResponse, UsersResponse, ClientSideIncrementalOffsetPaginatedResponse, @@ -27,12 +28,17 @@ INCREMENTAL_CURSOR_EXPORT_TYPES, ) +CHECKPOINT_INTERVAL = 1000 CURSOR_PAGINATION_PAGE_SIZE = 100 MAX_SATISFACTION_RATINGS_WINDOW_SIZE = timedelta(days=30) # Zendesk errors out if a start or end time parameter is 60 seconds or less in the past. TIME_PARAMETER_DELAY = timedelta(seconds=61) DATETIME_STRING_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +INCREMENTAL_TIME_EXPORT_REQ_PER_MIN_LIMIT = 10 + +incremental_time_export_api_lock = asyncio.Lock() + def url_base(subdomain: str) -> str: return f"https://{subdomain}.zendesk.com/api/v2" @@ -405,6 +411,123 @@ async def backfill_satisfaction_ratings( yield end +async def _fetch_incremental_time_export_resources( + http: HTTPSession, + subdomain: str, + name: str, + path: str, + response_model: type[IncrementalTimeExportResponse], + start_date: datetime, + log: Logger, +) -> AsyncGenerator[TimestampedResource | datetime, None]: + # Docs: https://developer.zendesk.com/documentation/ticketing/managing-tickets/using-the-incremental-export-api/#time-based-incremental-exports + # Incremental time export streams use timestamps for pagination that correlate to the updated_at timestamp for each record. + # The end_time returned in the response for fetching the next page is *always* the updated_at timestamp of the last + # record in the current response. This means that we'll always get at least one duplicate result when paginating. This also means + # that the stream could get stuck looping & making the same request if more than 1,000 results are updated at the same time, but + # an error should be raised if we detect that. + url = f"{url_base(subdomain)}/incremental/{path}" + + params = {"start_time": _dt_to_s(start_date)} + + last_seen_dt = start_date + count = 0 + + while True: + async with incremental_time_export_api_lock: + processor = IncrementalJsonProcessor( + await http.request_stream(log, url, params=params), + f"{name}.item", + TimestampedResource, + response_model, + ) + + async for resource in processor: + # Ignore duplicate results that were yielded on the previous sweep. + if resource.updated_at <= start_date: + continue + + # Checkpoint previously yielded documents if we see a new updated_at value. + if ( + resource.updated_at > last_seen_dt and + last_seen_dt != start_date and + count >= CHECKPOINT_INTERVAL + ): + yield last_seen_dt + count = 0 + + yield resource + count += 1 + last_seen_dt = resource.updated_at + + remainder = processor.get_remainder() + + # Handle empty responses. Since the end_time used to get the next page always overlaps with at least one + # record on the previous page, we should only see empty responses if users don't have any organizations updated + # on or afterthe start date. + if remainder.count == 0 or remainder.end_time is None: + return + + # Error if 1000+ organizations have the same updated_at value. This stops the stream from + # looping & making the same request endlessly. If this happens, we can evaluate different strategies + # for users that hit this issue. + if params["start_time"] == remainder.end_time and remainder.count >= 1000: + raise RuntimeError(f"At least 1,000 organizations were updated at {remainder.end_time}, and this stream cannot progress without potentially missing data. Contact Estuary Support for help resolving this issue.") + + if remainder.end_of_stream: + # Checkpoint the last document(s) if there were any updated records in this sweep. + if last_seen_dt > start_date: + yield last_seen_dt + + return + + params["start_time"] = remainder.end_time + + # Sleep to avoid excessively hitting this endpoint's more restricting 10 req/min limit. + await asyncio.sleep(60 / INCREMENTAL_TIME_EXPORT_REQ_PER_MIN_LIMIT) + + +async def fetch_incremental_time_export_resources( + http: HTTPSession, + subdomain: str, + name: str, + path: str, + response_model: type[IncrementalTimeExportResponse], + log: Logger, + log_cursor: LogCursor, +) -> AsyncGenerator[TimestampedResource | LogCursor, None]: + assert isinstance(log_cursor, datetime) + + generator = _fetch_incremental_time_export_resources(http, subdomain, name, path, response_model, log_cursor, log) + + async for result in generator: + yield result + + +async def backfill_incremental_time_export_resources( + http: HTTPSession, + subdomain: str, + name: str, + path: str, + response_model: type[IncrementalTimeExportResponse], + log: Logger, + page: PageCursor, + cutoff: LogCursor, +) -> AsyncGenerator[TimestampedResource | PageCursor, None]: + assert isinstance(page, int) + assert isinstance(cutoff, datetime) + + generator = _fetch_incremental_time_export_resources(http, subdomain, name, path, response_model, _s_to_dt(page), log) + + async for result in generator: + if isinstance(result, datetime): + yield _dt_to_s(result) + elif result.updated_at > cutoff: + return + else: + yield result + + async def _fetch_incremental_cursor_export_resources( http: HTTPSession, subdomain: str, diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index b4d2b3c3f3..3119cfb087 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -130,6 +130,24 @@ class TimestampedResource(ZendeskResource): updated_at: AwareDatetime +class IncrementalTimeExportResponse(BaseModel, extra="allow"): + next_page: str | None + count: int + end_of_stream: bool + end_time: int | None + resources: list[TimestampedResource] + + +class OrganizationsResponse(IncrementalTimeExportResponse): + resources: list[TimestampedResource] = Field(alias="organizations") + +# Incremental time based export resources. +# Tuples contain the name, path, and response model for each resource. +INCREMENTAL_TIME_EXPORT_RESOURCES: list[tuple[str, str, type[IncrementalTimeExportResponse]]] = [ + ("organizations", "organizations", OrganizationsResponse), +] + + class IncrementalCursorExportResponse(BaseModel, extra="allow"): after_cursor: str | None end_of_stream: bool diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index 4266981cf1..c5c9ec27f3 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -17,6 +17,7 @@ FullRefreshCursorPaginatedResponse, FullRefreshResource, IncrementalCursorPaginatedResponse, + IncrementalTimeExportResponse, ResourceConfig, ResourceState, TimestampedResource, @@ -27,6 +28,7 @@ FULL_REFRESH_RESOURCES, FULL_REFRESH_OFFSET_PAGINATED_RESOURCES, FULL_REFRESH_CURSOR_PAGINATED_RESOURCES, + INCREMENTAL_TIME_EXPORT_RESOURCES, INCREMENTAL_CURSOR_EXPORT_RESOURCES, INCREMENTAL_CURSOR_EXPORT_TYPES, INCREMENTAL_CURSOR_PAGINATED_RESOURCES, @@ -35,6 +37,7 @@ ) from .api import ( backfill_audit_logs, + backfill_incremental_time_export_resources, backfill_incremental_cursor_export_resources, backfill_incremental_cursor_paginated_resources, backfill_satisfaction_ratings, @@ -43,6 +46,7 @@ fetch_audit_logs, fetch_client_side_incremental_offset_paginated_resources, fetch_client_side_incremental_cursor_paginated_resources, + fetch_incremental_time_export_resources, fetch_incremental_cursor_export_resources, fetch_incremental_cursor_paginated_resources, fetch_satisfaction_ratings, @@ -518,6 +522,66 @@ def open( return resources +def incremental_time_export_resources( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> list[common.Resource]: + + def open( + name: str, + path: str, + response_model: type[IncrementalTimeExportResponse], + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings, + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_changes=functools.partial( + fetch_incremental_time_export_resources, + http, + config.subdomain, + name, + path, + response_model, + ), + fetch_page=functools.partial( + backfill_incremental_time_export_resources, + http, + config.subdomain, + name, + path, + response_model, + ) + ) + + cutoff = datetime.now(tz=UTC) - TIME_PARAMETER_DELAY + + resources = [ + common.Resource( + name=name, + key=["/id"], + model=TimestampedResource, + open=functools.partial(open, name, path, response_model), + initial_state=ResourceState( + inc=ResourceState.Incremental(cursor=cutoff), + backfill=ResourceState.Backfill(cutoff=cutoff, next_page=_dt_to_s(config.start_date)) + ), + initial_config=ResourceConfig( + name=name, interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + for (name, path, response_model) in INCREMENTAL_TIME_EXPORT_RESOURCES + ] + + return resources + + def incremental_cursor_export_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -656,6 +720,7 @@ async def all_resources( *client_side_filtered_cursor_paginated_resources(log, http, config), satisfaction_ratings(log, http, config), *incremental_cursor_paginated_resources(log, http, config), + *incremental_time_export_resources(log, http, config), *incremental_cursor_export_resources(log, http, config), *ticket_child_resources(log, http, config), ] diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index 9fc725f692..ce9caf184e 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -75,6 +75,10 @@ captures: name: ticket_metric_events interval: PT5M target: acmeCo/ticket_metric_events + - resource: + name: organizations + interval: PT5M + target: acmeCo/organizations - resource: name: tickets interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index 1e4f9242d8..35693841df 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -184,6 +184,32 @@ "view_tickets": true } ], + [ + "acmeCo/organizations", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "created_at": "2024-07-26T15:41:00Z", + "deleted_at": null, + "details": "", + "domain_names": [], + "external_id": null, + "group_id": null, + "id": 28835714737428, + "name": "Estuary", + "notes": "", + "organization_fields": {}, + "shared_comments": false, + "shared_tickets": false, + "tags": [ + "test-tag" + ], + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/organizations/28835714737428.json" + } + ], [ "acmeCo/satisfaction_ratings", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 0b0d1c1294..59fadf41ad 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -447,6 +447,70 @@ "/id" ] }, + { + "recommendedName": "organizations", + "resourceConfig": { + "name": "organizations", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "satisfaction_ratings", "resourceConfig": { From 40b00bb65084acaebf18e6294c1f0b0a887e3b0a Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 10:31:59 -0500 Subject: [PATCH 07/12] source-zendesk-support-native: add `automations`, `posts`, and `topics` --- .../acmeCo/automations.schema.yaml | 41 ++++ .../acmeCo/flow.yaml | 12 ++ .../acmeCo/posts.schema.yaml | 41 ++++ .../acmeCo/topics.schema.yaml | 41 ++++ .../source_zendesk_support_native/models.py | 15 ++ source-zendesk-support-native/test.flow.yaml | 12 ++ .../snapshots__capture__capture.stdout.json | 90 ++++++++ .../snapshots__discover__capture.stdout.json | 192 ++++++++++++++++++ 8 files changed, 444 insertions(+) create mode 100644 source-zendesk-support-native/acmeCo/automations.schema.yaml create mode 100644 source-zendesk-support-native/acmeCo/posts.schema.yaml create mode 100644 source-zendesk-support-native/acmeCo/topics.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/automations.schema.yaml b/source-zendesk-support-native/acmeCo/automations.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/automations.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index ea2e891242..6d1817c760 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -4,6 +4,10 @@ collections: schema: audit_logs.schema.yaml key: - /id + acmeCo/automations: + schema: automations.schema.yaml + key: + - /id acmeCo/brands: schema: brands.schema.yaml key: @@ -32,6 +36,10 @@ collections: schema: organizations.schema.yaml key: - /id + acmeCo/posts: + schema: posts.schema.yaml + key: + - /id acmeCo/satisfaction_ratings: schema: satisfaction_ratings.schema.yaml key: @@ -84,6 +92,10 @@ collections: schema: tickets.schema.yaml key: - /id + acmeCo/topics: + schema: topics.schema.yaml + key: + - /id acmeCo/users: schema: users.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/posts.schema.yaml b/source-zendesk-support-native/acmeCo/posts.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/posts.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/acmeCo/topics.schema.yaml b/source-zendesk-support-native/acmeCo/topics.schema.yaml new file mode 100644 index 0000000000..86ae494ca2 --- /dev/null +++ b/source-zendesk-support-native/acmeCo/topics.schema.yaml @@ -0,0 +1,41 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer + updated_at: + format: date-time + title: Updated At + type: string +required: + - id + - updated_at +title: TimestampedResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 3119cfb087..84786c09c6 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -290,6 +290,10 @@ class ClientSideIncrementalCursorPaginatedResponse(FullRefreshCursorPaginatedRes resources: list[TimestampedResource] +class AutomationsResponse(ClientSideIncrementalCursorPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="automations") + + class BrandsResponse(ClientSideIncrementalCursorPaginatedResponse): resources: list[TimestampedResource] = Field(alias="brands") @@ -314,15 +318,26 @@ class OrganizationMembershipsResponse(ClientSideIncrementalCursorPaginatedRespon resources: list[TimestampedResource] = Field(alias="organization_memberships") +class PostsResponse(ClientSideIncrementalCursorPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="posts") + + +class TopicsResponse(ClientSideIncrementalCursorPaginatedResponse): + resources: list[TimestampedResource] = Field(alias="topics") + + # Incremental client side resources that are paginated with a cursor. # Tuples contain the name, path, any additional request query params, and response model for each resource. CLIENT_SIDE_FILTERED_CURSOR_PAGINATED_RESOURCES: list[tuple[str, str, dict[str, str | int] | None, type[ClientSideIncrementalCursorPaginatedResponse]]] = [ + ("automations", "automations", None, AutomationsResponse), ("brands", "brands", None, BrandsResponse), ("groups", "groups", {"exclude_deleted": "false"}, GroupsResponse), ("group_memberships", "group_memberships", None, GroupMembershipsResponse), ("macros", "macros", None, MacrosResponse), ("organization_memberships", "organization_memberships", None, OrganizationMembershipsResponse), + ("posts", "community/posts", None, PostsResponse), ("ticket_fields", "ticket_fields", None, TicketFieldsResponse), + ("topics", "community/topics", None, TopicsResponse), ] diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index ce9caf184e..9175dcec02 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -39,6 +39,10 @@ captures: name: ticket_forms interval: PT5M target: acmeCo/ticket_forms + - resource: + name: automations + interval: PT5M + target: acmeCo/automations - resource: name: brands interval: PT5M @@ -59,10 +63,18 @@ captures: name: organization_memberships interval: PT5M target: acmeCo/organization_memberships + - resource: + name: posts + interval: PT5M + target: acmeCo/posts - resource: name: ticket_fields interval: PT5M target: acmeCo/ticket_fields + - resource: + name: topics + interval: PT5M + target: acmeCo/topics - resource: name: satisfaction_ratings interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index 35693841df..bd0520ecb8 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -1,4 +1,43 @@ [ + [ + "acmeCo/automations", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "actions": [ + { + "field": "status", + "value": "closed" + } + ], + "active": true, + "conditions": { + "all": [ + { + "field": "status", + "operator": "is", + "value": "solved" + }, + { + "field": "SOLVED", + "operator": "greater_than", + "value": "72" + } + ], + "any": [] + }, + "created_at": "2024-07-26T15:41:01Z", + "default": true, + "id": 28835714782868, + "position": 0, + "raw_title": "Close ticket 3 days after status is set to solved", + "title": "Close ticket 3 days after status is set to solved", + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/automations/28835714782868.json" + } + ], [ "acmeCo/brands", { @@ -210,6 +249,36 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/organizations/28835714737428.json" } ], + [ + "acmeCo/posts", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "author_id": 28835702984212, + "closed": false, + "comment_count": 2, + "content_tag_ids": [], + "created_at": "2024-07-26T20:41:46Z", + "details": "

You can add a topic like this one in your community. End users can add feature requests and describe their use cases. Other users can comment on the requests and vote for them. Product managers can review feature requests and provide feedback.

", + "featured": false, + "follower_count": 1, + "frozen": false, + "html_url": "https://d3v-estuary.zendesk.com/hc/en-us/community/posts/28847424303764-I-d-like-a-way-for-users-to-submit-feature-requests", + "id": 28847424303764, + "non_author_editor_id": null, + "non_author_updated_at": null, + "pinned": false, + "status": "none", + "title": "I'd like a way for users to submit feature requests", + "topic_id": 28847452157844, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/help_center/community/posts/28847424303764-I-d-like-a-way-for-users-to-submit-feature-requests.json", + "vote_count": 1, + "vote_sum": 1 + } + ], [ "acmeCo/satisfaction_ratings", { @@ -782,6 +851,27 @@ } } ], + [ + "acmeCo/topics", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "community_id": 28847452144020, + "created_at": "2024-07-26T20:41:45Z", + "description": null, + "follower_count": 1, + "html_url": "https://d3v-estuary.zendesk.com/hc/en-us/community/topics/28847452144788-General-Discussion", + "id": 28847452144788, + "manageable_by": "managers", + "name": "General Discussion", + "position": 0, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/help_center/community/topics/28847452144788.json", + "user_segment_id": null + } + ], [ "acmeCo/users", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 59fadf41ad..dc306d7364 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -63,6 +63,70 @@ "/id" ] }, + { + "recommendedName": "automations", + "resourceConfig": { + "name": "automations", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "brands", "resourceConfig": { @@ -511,6 +575,70 @@ "/id" ] }, + { + "recommendedName": "posts", + "resourceConfig": { + "name": "posts", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "satisfaction_ratings", "resourceConfig": { @@ -1204,6 +1332,70 @@ "/id" ] }, + { + "recommendedName": "topics", + "resourceConfig": { + "name": "topics", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "updated_at" + ], + "title": "TimestampedResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "users", "resourceConfig": { From cdee1011355a1bda013cf97acba024172b403297 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 10:50:56 -0500 Subject: [PATCH 08/12] source-zendesk-support-native: add `account_attributes` stream --- .../acmeCo/account_attributes.schema.yaml | 31 +++++++++++ .../acmeCo/flow.yaml | 4 ++ .../source_zendesk_support_native/models.py | 7 ++- source-zendesk-support-native/test.flow.yaml | 4 ++ .../snapshots__capture__capture.stdout.json | 15 ++++++ .../snapshots__discover__capture.stdout.json | 51 +++++++++++++++++++ 6 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 source-zendesk-support-native/acmeCo/account_attributes.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/account_attributes.schema.yaml b/source-zendesk-support-native/acmeCo/account_attributes.schema.yaml new file mode 100644 index 0000000000..f5a89babde --- /dev/null +++ b/source-zendesk-support-native/acmeCo/account_attributes.schema.yaml @@ -0,0 +1,31 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata +title: FullRefreshResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index 6d1817c760..cadfb5ddc3 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -1,5 +1,9 @@ --- collections: + acmeCo/account_attributes: + schema: account_attributes.schema.yaml + key: + - /_meta/row_id acmeCo/audit_logs: schema: audit_logs.schema.yaml key: diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 84786c09c6..542fd413af 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -189,6 +189,10 @@ class FullRefreshOffsetPaginatedResponse(FullRefreshResponse): next_page: str | None +class AccountAttributesResponse(FullRefreshOffsetPaginatedResponse): + resources: list[FullRefreshResource] = Field(alias="attributes") + + class SlaPoliciesResponse(FullRefreshOffsetPaginatedResponse): resources: list[FullRefreshResource] = Field(alias="sla_policies") @@ -196,7 +200,8 @@ class SlaPoliciesResponse(FullRefreshOffsetPaginatedResponse): # Full refresh resources that paginted with page offsets. # Tuples contain the name, path, and response model for each resource. FULL_REFRESH_OFFSET_PAGINATED_RESOURCES: list[tuple[str, str, type[FullRefreshOffsetPaginatedResponse]]] = [ - ("sla_policies", "slas/policies", SlaPoliciesResponse), + ("account_attributes", "routing/attributes", AccountAttributesResponse), + ("sla_policies", "slas/policies", SlaPoliciesResponse), ] diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index 9175dcec02..0b7cd019c7 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -23,6 +23,10 @@ captures: name: schedules interval: PT5M target: acmeCo/schedules + - resource: + name: account_attributes + interval: PT5M + target: acmeCo/account_attributes - resource: name: sla_policies interval: PT5M diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index bd0520ecb8..f119e593d9 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -1,4 +1,19 @@ [ + [ + "acmeCo/account_attributes", + { + "_meta": { + "op": "c", + "row_id": 0, + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef" + }, + "created_at": "2024-08-01T12:48:41Z", + "id": "6149ea30-5004-11ef-ba4d-1fb32c16a1d3", + "name": "Test attribute", + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/routing/attributes/6149ea30-5004-11ef-ba4d-1fb32c16a1d3.json" + } + ], [ "acmeCo/automations", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index dc306d7364..7538d11934 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -1,4 +1,55 @@ [ + { + "recommendedName": "account_attributes", + "resourceConfig": { + "name": "account_attributes", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + } + }, + "title": "FullRefreshResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, { "recommendedName": "audit_logs", "resourceConfig": { From 24b6c18d42b239dd8d6f7b8ec7d3078260edb849 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 12:00:06 -0500 Subject: [PATCH 09/12] source-zendesk-support-native: add `post_comments` and `post_votes` streams --- .../acmeCo/flow.yaml | 8 ++ .../acmeCo/post_comments.schema.yaml | 36 ++++++ .../acmeCo/post_votes.schema.yaml | 36 ++++++ .../source_zendesk_support_native/api.py | 36 ++++++ .../source_zendesk_support_native/models.py | 23 +++- .../resources.py | 50 ++++++++ source-zendesk-support-native/test.flow.yaml | 8 ++ .../snapshots__capture__capture.stdout.json | 40 ++++++ .../snapshots__discover__capture.stdout.json | 116 ++++++++++++++++++ 9 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 source-zendesk-support-native/acmeCo/post_comments.schema.yaml create mode 100644 source-zendesk-support-native/acmeCo/post_votes.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index cadfb5ddc3..e4d2bf58aa 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -40,6 +40,14 @@ collections: schema: organizations.schema.yaml key: - /id + acmeCo/post_comments: + schema: post_comments.schema.yaml + key: + - /id + acmeCo/post_votes: + schema: post_votes.schema.yaml + key: + - /id acmeCo/posts: schema: posts.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/post_comments.schema.yaml b/source-zendesk-support-native/acmeCo/post_comments.schema.yaml new file mode 100644 index 0000000000..31b68bb81b --- /dev/null +++ b/source-zendesk-support-native/acmeCo/post_comments.schema.yaml @@ -0,0 +1,36 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer +required: + - id +title: ZendeskResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/acmeCo/post_votes.schema.yaml b/source-zendesk-support-native/acmeCo/post_votes.schema.yaml new file mode 100644 index 0000000000..31b68bb81b --- /dev/null +++ b/source-zendesk-support-native/acmeCo/post_votes.schema.yaml @@ -0,0 +1,36 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer +required: + - id +title: ZendeskResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index 14e472f9f8..33bd462236 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -25,6 +25,8 @@ SatisfactionRatingsResponse, AuditLog, AuditLogsResponse, + Post, + PostsResponse, INCREMENTAL_CURSOR_EXPORT_TYPES, ) @@ -921,3 +923,37 @@ async def backfill_ticket_metrics( yield ZendeskResource.model_validate(metrics) else: return + + +async def fetch_post_child_resources( + http: HTTPSession, + subdomain: str, + path_segment: str, + response_model: type[IncrementalCursorPaginatedResponse], + log: Logger, + log_cursor: LogCursor, +) -> AsyncGenerator[ZendeskResource | LogCursor, None]: + assert isinstance(log_cursor, datetime) + + posts_generator = fetch_client_side_incremental_cursor_paginated_resources(http, subdomain, "community/posts", None, PostsResponse, log, log_cursor) + + async for result in posts_generator: + if isinstance(result, TimestampedResource): + post = Post.model_validate(result) + + if ( + (path_segment == "votes" and post.vote_count == 0) or + (path_segment == "comments" and post.comment_count == 0) + ): + continue + + path = f"community/posts/{post.id}/{path_segment}" + + async for child_resource in snapshot_cursor_paginated_resources(http, subdomain, path, response_model, log): + yield ZendeskResource.model_validate({ + "post_id": post.id, + **child_resource.model_dump(exclude={"meta_"}), + }) + + else: + yield result diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 542fd413af..50545fa2da 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -323,8 +323,13 @@ class OrganizationMembershipsResponse(ClientSideIncrementalCursorPaginatedRespon resources: list[TimestampedResource] = Field(alias="organization_memberships") +class Post(TimestampedResource): + comment_count: int + vote_count: int + + class PostsResponse(ClientSideIncrementalCursorPaginatedResponse): - resources: list[TimestampedResource] = Field(alias="posts") + resources: list[Post] = Field(alias="posts") class TopicsResponse(ClientSideIncrementalCursorPaginatedResponse): @@ -346,6 +351,22 @@ class TopicsResponse(ClientSideIncrementalCursorPaginatedResponse): ] +class PostVotesResponse(IncrementalCursorPaginatedResponse): + resources: list[ZendeskResource] = Field(alias="votes") + + +class PostCommentsResponse(IncrementalCursorPaginatedResponse): + resources: list[ZendeskResource] = Field(alias="comments") + + +# Resources that are fetched by following the posts stream & fetching resources for updated posts in a separate request. +# Tuples contain the name, path segment, and response model for each resource. +POST_CHILD_RESOURCES: list[tuple[str, str, type[IncrementalCursorPaginatedResponse]]] = [ + ("post_votes", "votes", PostVotesResponse), + ("post_comments", "comments", PostCommentsResponse), +] + + class AuditLog(ZendeskResource): created_at: AwareDatetime diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index c5c9ec27f3..7bcc0103e9 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -34,6 +34,7 @@ INCREMENTAL_CURSOR_PAGINATED_RESOURCES, OAUTH2_SPEC, TICKET_CHILD_RESOURCES, + POST_CHILD_RESOURCES, ) from .api import ( backfill_audit_logs, @@ -49,6 +50,7 @@ fetch_incremental_time_export_resources, fetch_incremental_cursor_export_resources, fetch_incremental_cursor_paginated_resources, + fetch_post_child_resources, fetch_satisfaction_ratings, fetch_ticket_child_resources, fetch_ticket_metrics, @@ -705,6 +707,53 @@ def open( return resources +def post_child_resources( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> list[common.Resource]: + + def open( + path_segment: str, + response_model: type[IncrementalCursorPaginatedResponse], + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings, + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_changes=functools.partial( + fetch_post_child_resources, + http, + config.subdomain, + path_segment, + response_model, + ), + ) + + resources = [ + common.Resource( + name=name, + key=["/id"], + model=ZendeskResource, + open=functools.partial(open, path_segment, response_model), + initial_state=ResourceState( + inc=ResourceState.Incremental(cursor=config.start_date), + ), + initial_config=ResourceConfig( + name=name, interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + for (name, path_segment, response_model) in POST_CHILD_RESOURCES + ] + + return resources + + async def all_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -723,4 +772,5 @@ async def all_resources( *incremental_time_export_resources(log, http, config), *incremental_cursor_export_resources(log, http, config), *ticket_child_resources(log, http, config), + *post_child_resources(log, http, config), ] diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index 0b7cd019c7..c76e382e21 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -111,3 +111,11 @@ captures: name: ticket_comments interval: PT5M target: acmeCo/ticket_comments + - resource: + name: post_votes + interval: PT5M + target: acmeCo/post_votes + - resource: + name: post_comments + interval: PT5M + target: acmeCo/post_comments diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index f119e593d9..e5fbb9f3c7 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -264,6 +264,46 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/organizations/28835714737428.json" } ], + [ + "acmeCo/post_comments", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "author_id": 28835702984212, + "body": "A test post comment.", + "created_at": "2024-08-01T13:10:11Z", + "html_url": "https://d3v-estuary.zendesk.com/hc/en-us/community/posts/28847424303764-I-d-like-a-way-for-users-to-submit-feature-requests/comments/29005581511700", + "id": 29005581511700, + "non_author_editor_id": null, + "non_author_updated_at": null, + "official": false, + "post_id": 28847424303764, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/help_center/community/posts/28847424303764/comments/29005581511700.json", + "vote_count": 1, + "vote_sum": 1 + } + ], + [ + "acmeCo/post_votes", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "created_at": "2024-08-01T13:15:00Z", + "id": 29005734783508, + "item_id": 28847424303764, + "item_type": "Post", + "post_id": 28847424303764, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/help_center/votes/29005734783508.json", + "user_id": 28835702984212, + "value": 1 + } + ], [ "acmeCo/posts", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 7538d11934..eaf61dea66 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -626,6 +626,122 @@ "/id" ] }, + { + "recommendedName": "post_comments", + "resourceConfig": { + "name": "post_comments", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + } + }, + "required": [ + "id" + ], + "title": "ZendeskResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "post_votes", + "resourceConfig": { + "name": "post_votes", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + } + }, + "required": [ + "id" + ], + "title": "ZendeskResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "posts", "resourceConfig": { From bb7cfc2b36fc723651af567b38d04980e3372a25 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 13:41:43 -0500 Subject: [PATCH 10/12] source-zendesk-support-native: add `post_comment_votes` stream --- .../acmeCo/flow.yaml | 4 ++ .../acmeCo/post_comment_votes.schema.yaml | 36 ++++++++++++ .../source_zendesk_support_native/api.py | 33 +++++++++++ .../source_zendesk_support_native/models.py | 11 +++- .../resources.py | 40 +++++++++++++ source-zendesk-support-native/test.flow.yaml | 4 ++ .../snapshots__capture__capture.stdout.json | 18 ++++++ .../snapshots__discover__capture.stdout.json | 58 +++++++++++++++++++ 8 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 source-zendesk-support-native/acmeCo/post_comment_votes.schema.yaml diff --git a/source-zendesk-support-native/acmeCo/flow.yaml b/source-zendesk-support-native/acmeCo/flow.yaml index e4d2bf58aa..6fc49b2a76 100644 --- a/source-zendesk-support-native/acmeCo/flow.yaml +++ b/source-zendesk-support-native/acmeCo/flow.yaml @@ -40,6 +40,10 @@ collections: schema: organizations.schema.yaml key: - /id + acmeCo/post_comment_votes: + schema: post_comment_votes.schema.yaml + key: + - /id acmeCo/post_comments: schema: post_comments.schema.yaml key: diff --git a/source-zendesk-support-native/acmeCo/post_comment_votes.schema.yaml b/source-zendesk-support-native/acmeCo/post_comment_votes.schema.yaml new file mode 100644 index 0000000000..31b68bb81b --- /dev/null +++ b/source-zendesk-support-native/acmeCo/post_comment_votes.schema.yaml @@ -0,0 +1,36 @@ +--- +$defs: + Meta: + properties: + op: + default: u + description: "Operation type (c: Create, u: Update, d: Delete)" + enum: + - c + - u + - d + title: Op + type: string + row_id: + default: -1 + description: "Row ID of the Document, counting up from zero, or -1 if not known" + title: Row Id + type: integer + title: Meta + type: object +additionalProperties: true +properties: + _meta: + $ref: "#/$defs/Meta" + default: + op: u + row_id: -1 + description: Document metadata + id: + title: Id + type: integer +required: + - id +title: ZendeskResource +type: object +x-infer-schema: true diff --git a/source-zendesk-support-native/source_zendesk_support_native/api.py b/source-zendesk-support-native/source_zendesk_support_native/api.py index 33bd462236..1c2e0dfacc 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/api.py +++ b/source-zendesk-support-native/source_zendesk_support_native/api.py @@ -27,6 +27,9 @@ AuditLogsResponse, Post, PostsResponse, + PostComment, + PostCommentsResponse, + PostCommentVotesResponse, INCREMENTAL_CURSOR_EXPORT_TYPES, ) @@ -957,3 +960,33 @@ async def fetch_post_child_resources( else: yield result + + +async def fetch_post_comment_votes( + http: HTTPSession, + subdomain: str, + log: Logger, + log_cursor: LogCursor, +) -> AsyncGenerator[ZendeskResource | LogCursor, None]: + assert isinstance(log_cursor, datetime) + + post_comments_generator = fetch_post_child_resources(http, subdomain, "comments", PostCommentsResponse, log, log_cursor) + + async for result in post_comments_generator: + + if isinstance(result, ZendeskResource): + post_comment = PostComment.model_validate(result.model_dump()) + + if post_comment.vote_count == 0: + continue + + path = f"community/posts/{post_comment.post_id}/comments/{post_comment.id}/votes" + + async for child_resource in snapshot_cursor_paginated_resources(http, subdomain, path, PostCommentVotesResponse, log): + yield ZendeskResource.model_validate({ + "post_id": post_comment.post_id, + **child_resource.model_dump(exclude={"meta_"}), + }) + + else: + yield result diff --git a/source-zendesk-support-native/source_zendesk_support_native/models.py b/source-zendesk-support-native/source_zendesk_support_native/models.py index 50545fa2da..6ee3a8fb2c 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/models.py +++ b/source-zendesk-support-native/source_zendesk_support_native/models.py @@ -355,8 +355,13 @@ class PostVotesResponse(IncrementalCursorPaginatedResponse): resources: list[ZendeskResource] = Field(alias="votes") +class PostComment(ZendeskResource): + post_id: int + vote_count: int + + class PostCommentsResponse(IncrementalCursorPaginatedResponse): - resources: list[ZendeskResource] = Field(alias="comments") + resources: list[PostComment] = Field(alias="comments") # Resources that are fetched by following the posts stream & fetching resources for updated posts in a separate request. @@ -373,3 +378,7 @@ class AuditLog(ZendeskResource): class AuditLogsResponse(FullRefreshCursorPaginatedResponse): resources: list[AuditLog] = Field(alias="audit_logs") + + +class PostCommentVotesResponse(PostVotesResponse): + pass diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index 7bcc0103e9..0e5a17f7f8 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -51,6 +51,7 @@ fetch_incremental_cursor_export_resources, fetch_incremental_cursor_paginated_resources, fetch_post_child_resources, + fetch_post_comment_votes, fetch_satisfaction_ratings, fetch_ticket_child_resources, fetch_ticket_metrics, @@ -754,6 +755,44 @@ def open( return resources +def post_comment_votes( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> common.Resource: + + def open( + binding: CaptureBinding[ResourceConfig], + binding_index: int, + state: ResourceState, + task: Task, + all_bindings, + ): + common.open_binding( + binding, + binding_index, + state, + task, + fetch_changes=functools.partial( + fetch_post_comment_votes, + http, + config.subdomain, + ), + ) + + return common.Resource( + name="post_comment_votes", + key=["/id"], + model=ZendeskResource, + open=open, + initial_state=ResourceState( + inc=ResourceState.Incremental(cursor=config.start_date), + ), + initial_config=ResourceConfig( + name="post_comment_votes", interval=timedelta(minutes=5) + ), + schema_inference=True, + ) + + async def all_resources( log: Logger, http: HTTPMixin, config: EndpointConfig ) -> list[common.Resource]: @@ -773,4 +812,5 @@ async def all_resources( *incremental_cursor_export_resources(log, http, config), *ticket_child_resources(log, http, config), *post_child_resources(log, http, config), + post_comment_votes(log, http, config), ] diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index c76e382e21..5b6ff0733e 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -119,3 +119,7 @@ captures: name: post_comments interval: PT5M target: acmeCo/post_comments + - resource: + name: post_comment_votes + interval: PT5M + target: acmeCo/post_comment_votes diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json index e5fbb9f3c7..e89d6aafb8 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__capture__capture.stdout.json @@ -264,6 +264,24 @@ "url": "https://d3v-estuary.zendesk.com/api/v2/organizations/28835714737428.json" } ], + [ + "acmeCo/post_comment_votes", + { + "_meta": { + "uuid": "DocUUIDPlaceholder-329Bb50aa48EAa9ef", + "row_id": 0 + }, + "created_at": "2024-08-01T13:19:33Z", + "id": 29005807715092, + "item_id": 29005581511700, + "item_type": "PostComment", + "post_id": 28847424303764, + "updated_at": "redacted", + "url": "https://d3v-estuary.zendesk.com/api/v2/help_center/votes/29005807715092.json", + "user_id": 28835702984212, + "value": 1 + } + ], [ "acmeCo/post_comments", { diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index eaf61dea66..5f88949bd9 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -626,6 +626,64 @@ "/id" ] }, + { + "recommendedName": "post_comment_votes", + "resourceConfig": { + "name": "post_comment_votes", + "interval": "PT5M" + }, + "documentSchema": { + "$defs": { + "Meta": { + "properties": { + "op": { + "default": "u", + "description": "Operation type (c: Create, u: Update, d: Delete)", + "enum": [ + "c", + "u", + "d" + ], + "title": "Op", + "type": "string" + }, + "row_id": { + "default": -1, + "description": "Row ID of the Document, counting up from zero, or -1 if not known", + "title": "Row Id", + "type": "integer" + } + }, + "title": "Meta", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "_meta": { + "$ref": "#/$defs/Meta", + "default": { + "op": "u", + "row_id": -1 + }, + "description": "Document metadata" + }, + "id": { + "title": "Id", + "type": "integer" + } + }, + "required": [ + "id" + ], + "title": "ZendeskResource", + "type": "object", + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, { "recommendedName": "post_comments", "resourceConfig": { From 1cdf9e7abeb32fdd8dee4cc75bdfb9ed2fb27a70 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 17:09:00 -0500 Subject: [PATCH 11/12] source-zendesk-support: increase default interval for infrequently updated resources --- .../resources.py | 24 ++++++------- source-zendesk-support-native/test.flow.yaml | 36 +++++++++---------- .../snapshots__discover__capture.stdout.json | 36 +++++++++---------- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index 0e5a17f7f8..c109fff088 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -216,7 +216,7 @@ def open( open=functools.partial(open, path, response_model), initial_state=ResourceState(), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=60) ), schema_inference=True, ) @@ -262,7 +262,7 @@ def open( open=functools.partial(open, path, response_model), initial_state=ResourceState(), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=60) ), schema_inference=True, ) @@ -291,11 +291,11 @@ def open( state, task, fetch_snapshot=functools.partial( - snapshot_cursor_paginated_resources, - http, - config.subdomain, - path, - response_model, + snapshot_cursor_paginated_resources, + http, + config.subdomain, + path, + response_model, ), tombstone=FullRefreshResource(_meta=FullRefreshResource.Meta(op="d")) ) @@ -308,7 +308,7 @@ def open( open=functools.partial(open, path, response_model), initial_state=ResourceState(), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=60) ), schema_inference=True, ) @@ -357,7 +357,7 @@ def open( inc=ResourceState.Incremental(cursor=EPOCH) ), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=30) ), schema_inference=True, ) @@ -408,7 +408,7 @@ def open( inc=ResourceState.Incremental(cursor=EPOCH) ), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=15) ), schema_inference=True, ) @@ -745,7 +745,7 @@ def open( inc=ResourceState.Incremental(cursor=config.start_date), ), initial_config=ResourceConfig( - name=name, interval=timedelta(minutes=5) + name=name, interval=timedelta(minutes=30) ), schema_inference=True, ) @@ -787,7 +787,7 @@ def open( inc=ResourceState.Incremental(cursor=config.start_date), ), initial_config=ResourceConfig( - name="post_comment_votes", interval=timedelta(minutes=5) + name="post_comment_votes", interval=timedelta(minutes=30) ), schema_inference=True, ) diff --git a/source-zendesk-support-native/test.flow.yaml b/source-zendesk-support-native/test.flow.yaml index 5b6ff0733e..ab2a45324c 100644 --- a/source-zendesk-support-native/test.flow.yaml +++ b/source-zendesk-support-native/test.flow.yaml @@ -21,63 +21,63 @@ captures: target: acmeCo/ticket_metrics - resource: name: schedules - interval: PT5M + interval: PT1H target: acmeCo/schedules - resource: name: account_attributes - interval: PT5M + interval: PT1H target: acmeCo/account_attributes - resource: name: sla_policies - interval: PT5M + interval: PT1H target: acmeCo/sla_policies - resource: name: tags - interval: PT5M + interval: PT1H target: acmeCo/tags - resource: name: custom_roles - interval: PT5M + interval: PT30M target: acmeCo/custom_roles - resource: name: ticket_forms - interval: PT5M + interval: PT30M target: acmeCo/ticket_forms - resource: name: automations - interval: PT5M + interval: PT15M target: acmeCo/automations - resource: name: brands - interval: PT5M + interval: PT15M target: acmeCo/brands - resource: name: groups - interval: PT5M + interval: PT15M target: acmeCo/groups - resource: name: group_memberships - interval: PT5M + interval: PT15M target: acmeCo/group_memberships - resource: name: macros - interval: PT5M + interval: PT15M target: acmeCo/macros - resource: name: organization_memberships - interval: PT5M + interval: PT15M target: acmeCo/organization_memberships - resource: name: posts - interval: PT5M + interval: PT15M target: acmeCo/posts - resource: name: ticket_fields - interval: PT5M + interval: PT15M target: acmeCo/ticket_fields - resource: name: topics - interval: PT5M + interval: PT15M target: acmeCo/topics - resource: name: satisfaction_ratings @@ -113,13 +113,13 @@ captures: target: acmeCo/ticket_comments - resource: name: post_votes - interval: PT5M + interval: PT30M target: acmeCo/post_votes - resource: name: post_comments - interval: PT5M + interval: PT30M target: acmeCo/post_comments - resource: name: post_comment_votes - interval: PT5M + interval: PT30M target: acmeCo/post_comment_votes diff --git a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json index 5f88949bd9..afeae88f9e 100644 --- a/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-zendesk-support-native/tests/snapshots/snapshots__discover__capture.stdout.json @@ -3,7 +3,7 @@ "recommendedName": "account_attributes", "resourceConfig": { "name": "account_attributes", - "interval": "PT5M" + "interval": "PT1H" }, "documentSchema": { "$defs": { @@ -118,7 +118,7 @@ "recommendedName": "automations", "resourceConfig": { "name": "automations", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -182,7 +182,7 @@ "recommendedName": "brands", "resourceConfig": { "name": "brands", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -246,7 +246,7 @@ "recommendedName": "custom_roles", "resourceConfig": { "name": "custom_roles", - "interval": "PT5M" + "interval": "PT30M" }, "documentSchema": { "$defs": { @@ -310,7 +310,7 @@ "recommendedName": "group_memberships", "resourceConfig": { "name": "group_memberships", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -374,7 +374,7 @@ "recommendedName": "groups", "resourceConfig": { "name": "groups", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -438,7 +438,7 @@ "recommendedName": "macros", "resourceConfig": { "name": "macros", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -502,7 +502,7 @@ "recommendedName": "organization_memberships", "resourceConfig": { "name": "organization_memberships", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -630,7 +630,7 @@ "recommendedName": "post_comment_votes", "resourceConfig": { "name": "post_comment_votes", - "interval": "PT5M" + "interval": "PT30M" }, "documentSchema": { "$defs": { @@ -688,7 +688,7 @@ "recommendedName": "post_comments", "resourceConfig": { "name": "post_comments", - "interval": "PT5M" + "interval": "PT30M" }, "documentSchema": { "$defs": { @@ -746,7 +746,7 @@ "recommendedName": "post_votes", "resourceConfig": { "name": "post_votes", - "interval": "PT5M" + "interval": "PT30M" }, "documentSchema": { "$defs": { @@ -804,7 +804,7 @@ "recommendedName": "posts", "resourceConfig": { "name": "posts", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -926,7 +926,7 @@ "recommendedName": "schedules", "resourceConfig": { "name": "schedules", - "interval": "PT5M" + "interval": "PT1H" }, "documentSchema": { "$defs": { @@ -977,7 +977,7 @@ "recommendedName": "sla_policies", "resourceConfig": { "name": "sla_policies", - "interval": "PT5M" + "interval": "PT1H" }, "documentSchema": { "$defs": { @@ -1028,7 +1028,7 @@ "recommendedName": "tags", "resourceConfig": { "name": "tags", - "interval": "PT5M" + "interval": "PT1H" }, "documentSchema": { "$defs": { @@ -1195,7 +1195,7 @@ "recommendedName": "ticket_fields", "resourceConfig": { "name": "ticket_fields", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { @@ -1259,7 +1259,7 @@ "recommendedName": "ticket_forms", "resourceConfig": { "name": "ticket_forms", - "interval": "PT5M" + "interval": "PT30M" }, "documentSchema": { "$defs": { @@ -1561,7 +1561,7 @@ "recommendedName": "topics", "resourceConfig": { "name": "topics", - "interval": "PT5M" + "interval": "PT15M" }, "documentSchema": { "$defs": { From f3ac472fad8b4d53057c5c2befa1feabca58176a Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Fri, 14 Feb 2025 17:37:36 -0500 Subject: [PATCH 12/12] source-zendesk-support-native: conditionally discover enterprise streams Some Zendesk accounts don't have access to `audit_logs` or `account_attributes`, so we don't discover them if receive a specific 403 error trying to hit the associated endpoint. There may be other streams that should be conditionally discovered, but they can be handled later when discovering them becomes an issue. --- .../resources.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/source-zendesk-support-native/source_zendesk_support_native/resources.py b/source-zendesk-support-native/source_zendesk_support_native/resources.py index c109fff088..3a5522b6d9 100644 --- a/source-zendesk-support-native/source_zendesk_support_native/resources.py +++ b/source-zendesk-support-native/source_zendesk_support_native/resources.py @@ -63,6 +63,24 @@ TIME_PARAMETER_DELAY, ) + +ENTERPRISE_STREAMS = ["audit_logs", "account_attributes"] + + +async def _is_enterprise_account( + log: Logger, http: HTTPMixin, config: EndpointConfig +) -> bool: + try: + await http.request(log, f"{url_base(config.subdomain)}/audit_logs") + except HTTPError as err: + if err.code == 403 and "You do not have access to this page." in err.message: + return False + else: + raise err + + return True + + async def validate_credentials( log: Logger, http: HTTPMixin, config: EndpointConfig ): @@ -798,7 +816,7 @@ async def all_resources( ) -> list[common.Resource]: http.token_source = TokenSource(oauth_spec=OAUTH2_SPEC, credentials=config.credentials) - return [ + resources = [ audit_logs(log, http, config), ticket_metrics(log, http, config), *full_refresh_resources(log, http, config), @@ -814,3 +832,8 @@ async def all_resources( *post_child_resources(log, http, config), post_comment_votes(log, http, config), ] + + if not await _is_enterprise_account(log, http, config): + resources = [r for r in resources if r.name not in ENTERPRISE_STREAMS] + + return resources