source-braintree-native: make non-transactions incremental streams …

…use `ResourceConfigWithSchedule`

Due to Braintree's limitation of only allowing queries for
`credit_card_verifications`, `customers`, `disputes`, and `subscriptions`
to filter based on their created_at datetime, we can't incrementally
capture updates to previously created data. We've been manually
backfilling these bindings every week, but now we can use the new
`ResourceConfigWithSchedule` resource config to set a schedule for
these bindings to automatically rebackfill.

I set the default schedule for these bindings to backfill every Friday
at 8:00 PM UTC, which seems like a reasonable setting for most use cases
that would prefer to perform possibly lengthy backfills over the weekend.

source-braintree-native/source_braintree_native/__init__.py

@@ -18,35 +18,35 @@
 from .models import (
     ConnectorState,
     EndpointConfig,
-    ResourceConfig,
+    ResourceConfigWithSchedule,
 )
 
 
 class Connector(
-    BaseCaptureConnector[EndpointConfig, ResourceConfig, ConnectorState],
+    BaseCaptureConnector[EndpointConfig, ResourceConfigWithSchedule, ConnectorState],
     HTTPMixin,
 ):
     def request_class(self):
-        return Request[EndpointConfig, ResourceConfig, ConnectorState]
+        return Request[EndpointConfig, ResourceConfigWithSchedule, ConnectorState]
 
     async def spec(self, _: request.Spec, logger: Logger) -> ConnectorSpec:
         return ConnectorSpec(
             configSchema=EndpointConfig.model_json_schema(),
             documentationUrl="https://go.estuary.dev/source-braintree-native",
-            resourceConfigSchema=ResourceConfig.model_json_schema(),
-            resourcePathPointers=ResourceConfig.PATH_POINTERS,
+            resourceConfigSchema=ResourceConfigWithSchedule.model_json_schema(),
+            resourcePathPointers=ResourceConfigWithSchedule.PATH_POINTERS,
         )
 
     async def discover(
         self, log: Logger, discover: request.Discover[EndpointConfig]
-    ) -> response.Discovered[ResourceConfig]:
+    ) -> response.Discovered[ResourceConfigWithSchedule]:
         resources = await all_resources(log, self, discover.config)
         return common.discovered(resources)
 
     async def validate(
         self,
         log: Logger,
-        validate: request.Validate[EndpointConfig, ResourceConfig],
+        validate: request.Validate[EndpointConfig, ResourceConfigWithSchedule],
     ) -> response.Validated:
         validate_credentials(log, validate.config)
         resources = await all_resources(log, self, validate.config)
@@ -56,7 +56,7 @@ async def validate(
     async def open(
         self,
         log: Logger,
-        open: request.Open[EndpointConfig, ResourceConfig, ConnectorState],
+        open: request.Open[EndpointConfig, ResourceConfigWithSchedule, ConnectorState],
     ) -> tuple[response.Opened, Callable[[Task], Awaitable[None]]]:
         resources = await all_resources(log, self, open.capture.config)
         resolved = common.resolve_bindings(open.capture.bindings, resources)

source-braintree-native/source_braintree_native/models.py

@@ -9,7 +9,7 @@
     ConnectorState as GenericConnectorState,
     LogCursor,
     PageCursor,
-    ResourceConfig,
+    ResourceConfigWithSchedule,
     ResourceState,
 )
 

source-braintree-native/source_braintree_native/resources.py

@@ -12,7 +12,7 @@
 
 from .models import (
     EndpointConfig,
-    ResourceConfig,
+    ResourceConfigWithSchedule,
     ResourceState,
     FullRefreshResource,
     IncrementalResource,
@@ -85,7 +85,7 @@ def open(
             gateway: BraintreeGateway,
             gateway_property: str,
             gateway_response_field: str | None,
-            binding: CaptureBinding[ResourceConfig],
+            binding: CaptureBinding[ResourceConfigWithSchedule],
             binding_index: int,
             state: ResourceState,
             task: Task,
@@ -112,7 +112,7 @@ def open(
             model=FullRefreshResource,
             open=functools.partial(open, _create_gateway(config), gateway_property, gateway_response_field),
             initial_state=ResourceState(),
-            initial_config=ResourceConfig(
+            initial_config=ResourceConfigWithSchedule(
                 name=name, interval=timedelta(minutes=5)
             ),
             schema_inference=True,
@@ -130,7 +130,7 @@ def open(
             fetch_page_fn: IncrementalResourceFetchPageFn,
             gateway: BraintreeGateway,
             window_size: int,
-            binding: CaptureBinding[ResourceConfig],
+            binding: CaptureBinding[ResourceConfigWithSchedule],
             binding_index: int,
             state: ResourceState,
             task: Task,
@@ -165,8 +165,8 @@ def open(
                 inc=ResourceState.Incremental(cursor=cutoff),
                 backfill=ResourceState.Backfill(next_page=_dt_to_str(config.start_date), cutoff=cutoff)
             ),
-            initial_config=ResourceConfig(
-                name=name, interval=timedelta(minutes=5)
+            initial_config=ResourceConfigWithSchedule(
+                name=name, interval=timedelta(minutes=5), schedule="0 20 * * 5",
             ),
             schema_inference=True,
         )
@@ -181,7 +181,7 @@ def transactions(
     def open(
             gateway: BraintreeGateway,
             window_size: int,
-            binding: CaptureBinding[ResourceConfig],
+            binding: CaptureBinding[ResourceConfigWithSchedule],
             binding_index: int,
             state: ResourceState,
             task: Task,
@@ -215,7 +215,7 @@ def open(
             inc=ResourceState.Incremental(cursor=cutoff),
             backfill=ResourceState.Backfill(next_page=_dt_to_str(config.start_date), cutoff=cutoff)
         ),
-        initial_config=ResourceConfig(
+        initial_config=ResourceConfigWithSchedule(
             name='transactions', interval=timedelta(minutes=5)
         ),
         schema_inference=True,

source-braintree-native/tests/snapshots/snapshots__discover__capture.stdout.json

@@ -207,7 +207,8 @@
     "recommendedName": "credit_card_verifications",
     "resourceConfig": {
       "name": "credit_card_verifications",
-      "interval": "PT5M"
+      "interval": "PT5M",
+      "schedule": "0 20 * * 5"
     },
     "documentSchema": {
       "$defs": {
@@ -271,7 +272,8 @@
     "recommendedName": "customers",
     "resourceConfig": {
       "name": "customers",
-      "interval": "PT5M"
+      "interval": "PT5M",
+      "schedule": "0 20 * * 5"
     },
     "documentSchema": {
       "$defs": {
@@ -335,7 +337,8 @@
     "recommendedName": "disputes",
     "resourceConfig": {
       "name": "disputes",
-      "interval": "PT5M"
+      "interval": "PT5M",
+      "schedule": "0 20 * * 5"
     },
     "documentSchema": {
       "$defs": {
@@ -399,7 +402,8 @@
     "recommendedName": "subscriptions",
     "resourceConfig": {
       "name": "subscriptions",
-      "interval": "PT5M"
+      "interval": "PT5M",
+      "schedule": "0 20 * * 5"
     },
     "documentSchema": {
       "$defs": {

source-braintree-native/tests/snapshots/snapshots__spec__capture.stdout.json

@@ -91,7 +91,6 @@
     },
     "resourceConfigSchema": {
       "additionalProperties": false,
-      "description": "ResourceConfig is a common resource configuration shape.",
       "properties": {
         "name": {
           "description": "Name of this resource",
@@ -104,12 +103,19 @@
           "format": "duration",
           "title": "Interval",
           "type": "string"
+        },
+        "schedule": {
+          "default": "",
+          "description": "Schedule to automatically rebackfill this binding. Accepts a cron expression.",
+          "pattern": "^((?:[0-5]?\\d(?:-[0-5]?\\d)?|\\*(?:/[0-5]?\\d)?)(?:,(?:[0-5]?\\d(?:-[0-5]?\\d)?|\\*(?:/[0-5]?\\d)?))*)\\s+((?:[01]?\\d|2[0-3]|(?:[01]?\\d|2[0-3])-(?:[01]?\\d|2[0-3])|\\*(?:/[01]?\\d|/2[0-3])?)(?:,(?:[01]?\\d|2[0-3]|(?:[01]?\\d|2[0-3])-(?:[01]?\\d|2[0-3])|\\*(?:/[01]?\\d|/2[0-3])?))*)\\s+((?:0?[1-9]|[12]\\d|3[01]|(?:0?[1-9]|[12]\\d|3[01])-(?:0?[1-9]|[12]\\d|3[01])|\\*(?:/[0-9]|/1[0-9]|/2[0-9]|/3[01])?)(?:,(?:0?[1-9]|[12]\\d|3[01]|(?:0?[1-9]|[12]\\d|3[01])-(?:0?[1-9]|[12]\\d|3[01])|\\*(?:/[0-9]|/1[0-9]|/2[0-9]|/3[01])?))*)\\s+((?:[1-9]|1[0-2]|(?:[1-9]|1[0-2])-(?:[1-9]|1[0-2])|\\*(?:/[1-9]|/1[0-2])?)(?:,(?:[1-9]|1[0-2]|(?:[1-9]|1[0-2])-(?:[1-9]|1[0-2])|\\*(?:/[1-9]|/1[0-2])?))*)\\s+((?:[0-6]|(?:[0-6])-(?:[0-6])|\\*(?:/[0-6])?)(?:,(?:[0-6]|(?:[0-6])-(?:[0-6])|\\*(?:/[0-6])?))*)$|^$",
+          "title": "Schedule",
+          "type": "string"
         }
       },
       "required": [
         "name"
       ],
-      "title": "ResourceConfig",
+      "title": "ResourceConfigWithSchedule",
       "type": "object"
     },
     "documentationUrl": "https://go.estuary.dev/source-braintree-native",