Skip to content

Commit

Permalink
source-bigquery-batch: Option 'Discover Views'
Browse files Browse the repository at this point in the history
  • Loading branch information
@willdonnelly
willdonnelly committed Feb 26, 2025
1 parent 1f1e7be commit eb7a1fc
Show file tree
Hide file tree
Showing 7 changed files with 357 additions and 15 deletions.
45 changes: 45 additions & 0 deletions source-bigquery-batch/.snapshots/TestCaptureFromView-Capture
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# ================================
# Collection "acmeCo/test/capturefromview_194890": 23 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"id":0,"name":"Row 0","updated_at":"2025-02-13T12:00:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"id":1,"name":"Row 1","updated_at":"2025-02-13T12:01:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"id":2,"name":"Row 2","updated_at":"2025-02-13T12:02:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"id":3,"name":"Row 3","updated_at":"2025-02-13T12:03:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"id":4,"name":"Row 4","updated_at":"2025-02-13T12:04:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"id":5,"name":"Row 5","updated_at":"2025-02-13T12:05:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"id":6,"name":"Row 6","updated_at":"2025-02-13T12:06:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"id":7,"name":"Row 7","updated_at":"2025-02-13T12:07:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"id":8,"name":"Row 8","updated_at":"2025-02-13T12:08:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"id":9,"name":"Row 9","updated_at":"2025-02-13T12:09:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":10},"id":10,"name":"Row 10","updated_at":"2025-02-13T12:10:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":11},"id":11,"name":"Row 11","updated_at":"2025-02-13T12:11:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":12},"id":12,"name":"Row 12","updated_at":"2025-02-13T12:12:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":13},"id":13,"name":"Row 13","updated_at":"2025-02-13T12:13:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":14},"id":14,"name":"Row 14","updated_at":"2025-02-13T12:14:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":15},"id":15,"name":"Row 15","updated_at":"2025-02-13T12:15:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":16},"id":16,"name":"Row 16","updated_at":"2025-02-13T12:16:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":17},"id":17,"name":"Row 17","updated_at":"2025-02-13T12:17:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":18},"id":18,"name":"Row 18","updated_at":"2025-02-13T12:18:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":19},"id":19,"name":"Row 19","updated_at":"2025-02-13T12:19:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":20},"id":3,"name":"Row 3","updated_at":"2025-02-13T12:20:00Z","visible":true}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":21},"id":4,"name":"Row 4","updated_at":"2025-02-13T12:20:00Z","visible":false}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":22},"id":2,"name":"Row 2","updated_at":"2025-02-13T12:20:00Z","visible":false}
# ================================
# Collection "acmeCo/test/capturefromview_227836": 11 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"id":0,"name":"Row 0","updated_at":"2025-02-13T12:00:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"id":2,"name":"Row 2","updated_at":"2025-02-13T12:02:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"id":4,"name":"Row 4","updated_at":"2025-02-13T12:04:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"id":6,"name":"Row 6","updated_at":"2025-02-13T12:06:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"id":8,"name":"Row 8","updated_at":"2025-02-13T12:08:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":5},"id":10,"name":"Row 10","updated_at":"2025-02-13T12:10:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":6},"id":12,"name":"Row 12","updated_at":"2025-02-13T12:12:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":7},"id":14,"name":"Row 14","updated_at":"2025-02-13T12:14:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":8},"id":16,"name":"Row 16","updated_at":"2025-02-13T12:16:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":9},"id":18,"name":"Row 18","updated_at":"2025-02-13T12:18:00Z"}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":10},"id":3,"name":"Row 3","updated_at":"2025-02-13T12:20:00Z"}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"capturefromview_194890":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13T12:20:00Z"],"DocumentCount":23,"LastPolled":"<TIMESTAMP>"},"capturefromview_227836":{"CursorNames":["updated_at"],"CursorValues":["2025-02-13T12:20:00Z"],"DocumentCount":11,"LastPolled":"<TIMESTAMP>"}}}

137 changes: 137 additions & 0 deletions source-bigquery-batch/.snapshots/TestCaptureFromView-DiscoveryWithViews
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
Binding 0:
{
"resource_config_json": {
"name": "capturefromview_194890",
"schema": "testdata",
"table": "capturefromview_194890"
},
"resource_path": [
"capturefromview_194890"
],
"collection": {
"name": "acmeCo/test/capturefromview_194890",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "capturefromview_194890"
}
Binding 1:
{
"resource_config_json": {
"name": "capturefromview_227836",
"schema": "testdata",
"table": "capturefromview_227836"
},
"resource_path": [
"capturefromview_227836"
],
"collection": {
"name": "acmeCo/test/capturefromview_227836",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "capturefromview_227836"
}

69 changes: 69 additions & 0 deletions source-bigquery-batch/.snapshots/TestCaptureFromView-DiscoveryWithoutViews
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
Binding 0:
{
"resource_config_json": {
"name": "capturefromview_194890",
"schema": "testdata",
"table": "capturefromview_194890"
},
"resource_path": [
"capturefromview_194890"
],
"collection": {
"name": "acmeCo/test/capturefromview_194890",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "capturefromview_194890"
}

5 changes: 5 additions & 0 deletions source-bigquery-batch/.snapshots/TestSpec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@
},
"advanced": {
"properties": {
"discover_views": {
"type": "boolean",
"title": "Discover Views",
"description": "When set views will be automatically discovered as resources. If unset only tables will be discovered."
},
"poll": {
"type": "string",
"title": "Default Polling Schedule",
Expand Down
4 changes: 2 additions & 2 deletions source-bigquery-batch/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ type BatchSQLDriver struct {
ConfigSchema json.RawMessage

Connect func(ctx context.Context, cfg *Config) (*bigquery.Client, error)
GenerateResource func(resourceName, schemaName, tableName, tableType string) (*Resource, error)
GenerateResource func(cfg *Config, resourceName, schemaName, tableName, tableType string) (*Resource, error)
ExcludedSystemSchemas []string
SelectQueryTemplate func(res *Resource) (string, error)
}
Expand Down Expand Up @@ -238,7 +238,7 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove
var tableID = table.Schema + "." + table.Name

var recommendedName = recommendedCatalogName(table.Name)
var res, err = drv.GenerateResource(recommendedName, table.Schema, table.Name, table.Type)
var res, err = drv.GenerateResource(&cfg, recommendedName, table.Schema, table.Name, table.Type)
if err != nil {
log.WithFields(log.Fields{
"reason": err,
Expand Down
29 changes: 18 additions & 11 deletions source-bigquery-batch/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ type Config struct {
}

type advancedConfig struct {
PollSchedule string `json:"poll,omitempty" jsonschema:"title=Default Polling Schedule,description=When and how often to execute fetch queries. Accepts a Go duration string like '5m' or '6h' for frequency-based polling or a string like 'daily at 12:34Z' to poll at a specific time (specified in UTC) every day. Defaults to '24h' if unset." jsonschema_extras:"pattern=^([-+]?([0-9]+([.][0-9]+)?(h|m|s|ms))+|daily at [0-9][0-9]?:[0-9]{2}Z)$"`
FeatureFlags string `json:"feature_flags,omitempty" jsonschema:"title=Feature Flags,description=This property is intended for Estuary internal use. You should only modify this field as directed by Estuary support."`
DiscoverViews bool `json:"discover_views,omitempty" jsonschema:"title=Discover Views,description=When set views will be automatically discovered as resources. If unset only tables will be discovered."`
PollSchedule string `json:"poll,omitempty" jsonschema:"title=Default Polling Schedule,description=When and how often to execute fetch queries. Accepts a Go duration string like '5m' or '6h' for frequency-based polling or a string like 'daily at 12:34Z' to poll at a specific time (specified in UTC) every day. Defaults to '24h' if unset." jsonschema_extras:"pattern=^([-+]?([0-9]+([.][0-9]+)?(h|m|s|ms))+|daily at [0-9][0-9]?:[0-9]{2}Z)$"`
FeatureFlags string `json:"feature_flags,omitempty" jsonschema:"title=Feature Flags,description=This property is intended for Estuary internal use. You should only modify this field as directed by Estuary support."`

parsedFeatureFlags map[string]bool // Parsed feature flags setting with defaults applied
}
Expand Down Expand Up @@ -168,16 +169,22 @@ func quoteIdentifier(name string) string {
return "`" + strings.ReplaceAll(name, "`", "\\`") + "`"
}

func generateBigQueryResource(resourceName, schemaName, tableName, tableType string) (*Resource, error) {
if !strings.EqualFold(tableType, "BASE TABLE") {
return nil, fmt.Errorf("discovery will not autogenerate resource configs for entities of type %q, but you may add them manually", tableType)
func generateBigQueryResource(cfg *Config, resourceName, schemaName, tableName, tableType string) (*Resource, error) {
if strings.EqualFold(tableType, "BASE TABLE") {
return &Resource{
Name: resourceName,
SchemaName: schemaName,
TableName: tableName,
}, nil
}

return &Resource{
Name: resourceName,
SchemaName: schemaName,
TableName: tableName,
}, nil
if strings.EqualFold(tableType, "VIEW") && cfg.Advanced.DiscoverViews {
return &Resource{
Name: resourceName,
SchemaName: schemaName,
TableName: tableName,
}, nil
}
return nil, fmt.Errorf("unsupported entity type %q", tableType)
}

var bigqueryDriver = &BatchSQLDriver{
Expand Down
Loading

0 comments on commit eb7a1fc

Please sign in to comment.