data.proto

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.events.cloud.datastream.v1;

import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Events.Protobuf.Cloud.Datastream.V1";
option php_namespace = "Google\\Events\\Cloud\\Datastream\\V1";
option ruby_package = "Google::Events::Cloud::Datastream::V1";

// Oracle database profile.
message OracleProfile {
  // Required. Hostname for the Oracle connection.
  string hostname = 1;

  // Port for the Oracle connection, default value is 1521.
  int32 port = 2;

  // Required. Username for the Oracle connection.
  string username = 3;

  // Required. Database for the Oracle connection.
  string database_service = 5;

  // Connection string attributes
  map<string, string> connection_attributes = 6;
}

// MySQL database profile.
message MysqlProfile {
  // Required. Hostname for the MySQL connection.
  string hostname = 1;

  // Port for the MySQL connection, default value is 3306.
  int32 port = 2;

  // Required. Username for the MySQL connection.
  string username = 3;

  // SSL configuration for the MySQL connection.
  MysqlSslConfig ssl_config = 5;
}

// PostgreSQL database profile.
message PostgresqlProfile {
  // Required. Hostname for the PostgreSQL connection.
  string hostname = 1;

  // Port for the PostgreSQL connection, default value is 5432.
  int32 port = 2;

  // Required. Username for the PostgreSQL connection.
  string username = 3;

  // Required. Database for the PostgreSQL connection.
  string database = 5;
}

// Cloud Storage bucket profile.
message GcsProfile {
  // Required. The Cloud Storage bucket name.
  string bucket = 1;

  // The root path inside the Cloud Storage bucket.
  string root_path = 2;
}

// BigQuery warehouse profile.
message BigQueryProfile {

}

// Static IP address connectivity.
message StaticServiceIpConnectivity {

}

// Forward SSH Tunnel connectivity.
message ForwardSshTunnelConnectivity {
  // Required. Hostname for the SSH tunnel.
  string hostname = 1;

  // Required. Username for the SSH tunnel.
  string username = 2;

  // Port for the SSH tunnel, default value is 22.
  int32 port = 3;
}

// The VPC Peering configuration is used to create VPC peering between
// Datastream and the consumer's VPC.
message VpcPeeringConfig {
  // Required. Fully qualified name of the VPC that Datastream will peer to.
  // Format: `projects/{project}/global/{networks}/{name}`
  string vpc = 1;

  // Required. A free subnet for peering. (CIDR of /29)
  string subnet = 2;
}

// The PrivateConnection resource is used to establish private connectivity
// between Datastream and a customer's network.
message PrivateConnection {
  // Private Connection state.
  enum State {
    // Unspecified state.
    STATE_UNSPECIFIED = 0;

    // The private connection is in creation state - creating resources.
    CREATING = 1;

    // The private connection has been created with all of its resources.
    CREATED = 2;

    // The private connection creation has failed.
    FAILED = 3;

    // The private connection is being deleted.
    DELETING = 4;

    // Delete request has failed, resource is in invalid state.
    FAILED_TO_DELETE = 5;
  }

  // Output only. The resource's name.
  string name = 1;

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2;

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3;

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5;

  // Output only. The state of the Private Connection.
  State state = 6;

  // Output only. In case of error, the details of the error in a user-friendly format.
  Error error = 7;

  // VPC Peering Config.
  VpcPeeringConfig vpc_peering_config = 100;
}

// Private Connectivity
message PrivateConnectivity {
  // Required. A reference to a private connection resource.
  // Format: `projects/{project}/locations/{location}/privateConnections/{name}`
  string private_connection = 1;
}

// The route resource is the child of the private connection resource,
// used for defining a route for a private connection.
message Route {
  // Output only. The resource's name.
  string name = 1;

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2;

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3;

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5;

  // Required. Destination address for connection
  string destination_address = 6;

  // Destination port for connection
  int32 destination_port = 7;
}

// MySQL SSL configuration information.
message MysqlSslConfig {
  // Output only. Indicates whether the client_key field is set.
  bool client_key_set = 2;

  // Output only. Indicates whether the client_certificate field is set.
  bool client_certificate_set = 4;

  // Output only. Indicates whether the ca_certificate field is set.
  bool ca_certificate_set = 6;
}

// A set of reusable connection configurations to be used as a source or
// destination for a stream.
message ConnectionProfile {
  // Output only. The resource's name.
  string name = 1;

  // Output only. The create time of the resource.
  google.protobuf.Timestamp create_time = 2;

  // Output only. The update time of the resource.
  google.protobuf.Timestamp update_time = 3;

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5;

  // Connection configuration for the ConnectionProfile.
  oneof profile {
    // Oracle ConnectionProfile configuration.
    OracleProfile oracle_profile = 100;

    // Cloud Storage ConnectionProfile configuration.
    GcsProfile gcs_profile = 101;

    // MySQL ConnectionProfile configuration.
    MysqlProfile mysql_profile = 102;

    // BigQuery Connection Profile configuration.
    BigQueryProfile bigquery_profile = 103;

    // PostgreSQL Connection Profile configuration.
    PostgresqlProfile postgresql_profile = 104;
  }

  // Connectivity options used to establish a connection to the profile.
  oneof connectivity {
    // Static Service IP connectivity.
    StaticServiceIpConnectivity static_service_ip_connectivity = 200;

    // Forward SSH tunnel connectivity.
    ForwardSshTunnelConnectivity forward_ssh_connectivity = 201;

    // Private connectivity.
    PrivateConnectivity private_connectivity = 202;
  }
}

// Oracle Column.
message OracleColumn {
  // Column name.
  string column = 1;

  // The Oracle data type.
  string data_type = 2;

  // Column length.
  int32 length = 3;

  // Column precision.
  int32 precision = 4;

  // Column scale.
  int32 scale = 5;

  // Column encoding.
  string encoding = 6;

  // Whether or not the column represents a primary key.
  bool primary_key = 7;

  // Whether or not the column can accept a null value.
  bool nullable = 8;

  // The ordinal position of the column in the table.
  int32 ordinal_position = 9;
}

// Oracle table.
message OracleTable {
  // Table name.
  string table = 1;

  // Oracle columns in the schema.
  // When unspecified as part of include/exclude objects, includes/excludes
  // everything.
  repeated OracleColumn oracle_columns = 2;
}

// Oracle schema.
message OracleSchema {
  // Schema name.
  string schema = 1;

  // Tables in the schema.
  repeated OracleTable oracle_tables = 2;
}

// Oracle database structure.
message OracleRdbms {
  // Oracle schemas/databases in the database server.
  repeated OracleSchema oracle_schemas = 1;
}

// Oracle data source configuration
message OracleSourceConfig {
  // Configuration to drop large object values.
  message DropLargeObjects {

  }

  // Configuration to stream large object values.
  message StreamLargeObjects {

  }

  // Oracle objects to include in the stream.
  OracleRdbms include_objects = 1;

  // Oracle objects to exclude from the stream.
  OracleRdbms exclude_objects = 2;

  // Maximum number of concurrent CDC tasks. The number should be non negative.
  // If not set (or set to 0), the system's default value will be used.
  int32 max_concurrent_cdc_tasks = 3;

  // The configuration for handle Oracle large objects.
  oneof large_objects_handling {
    // Drop large object values.
    DropLargeObjects drop_large_objects = 100;

    // Stream large object values. NOTE: This feature is currently experimental.
    StreamLargeObjects stream_large_objects = 102;
  }
}

// PostgreSQL Column.
message PostgresqlColumn {
  // Column name.
  string column = 1;

  // The PostgreSQL data type.
  string data_type = 2;

  // Column length.
  int32 length = 3;

  // Column precision.
  int32 precision = 4;

  // Column scale.
  int32 scale = 5;

  // Whether or not the column represents a primary key.
  bool primary_key = 7;

  // Whether or not the column can accept a null value.
  bool nullable = 8;

  // The ordinal position of the column in the table.
  int32 ordinal_position = 9;
}

// PostgreSQL table.
message PostgresqlTable {
  // Table name.
  string table = 1;

  // PostgreSQL columns in the schema.
  // When unspecified as part of include/exclude objects,
  // includes/excludes everything.
  repeated PostgresqlColumn postgresql_columns = 2;
}

// PostgreSQL schema.
message PostgresqlSchema {
  // Schema name.
  string schema = 1;

  // Tables in the schema.
  repeated PostgresqlTable postgresql_tables = 2;
}

// PostgreSQL database structure.
message PostgresqlRdbms {
  // PostgreSQL schemas in the database server.
  repeated PostgresqlSchema postgresql_schemas = 1;
}

// PostgreSQL data source configuration
message PostgresqlSourceConfig {
  // PostgreSQL objects to include in the stream.
  PostgresqlRdbms include_objects = 1;

  // PostgreSQL objects to exclude from the stream.
  PostgresqlRdbms exclude_objects = 2;

  // Required. Immutable. The name of the logical replication slot that's configured with the
  // pgoutput plugin.
  string replication_slot = 3;

  // Required. The name of the publication that includes the set of all tables that are
  // defined in the stream's include_objects.
  string publication = 4;
}

// MySQL Column.
message MysqlColumn {
  // Column name.
  string column = 1;

  // The MySQL data type. Full data types list can be found here:
  // https://dev.mysql.com/doc/refman/8.0/en/data-types.html
  string data_type = 2;

  // Column length.
  int32 length = 3;

  // Column collation.
  string collation = 4;

  // Whether or not the column represents a primary key.
  bool primary_key = 5;

  // Whether or not the column can accept a null value.
  bool nullable = 6;

  // The ordinal position of the column in the table.
  int32 ordinal_position = 7;
}

// MySQL table.
message MysqlTable {
  // Table name.
  string table = 1;

  // MySQL columns in the database.
  // When unspecified as part of include/exclude objects, includes/excludes
  // everything.
  repeated MysqlColumn mysql_columns = 2;
}

// MySQL database.
message MysqlDatabase {
  // Database name.
  string database = 1;

  // Tables in the database.
  repeated MysqlTable mysql_tables = 2;
}

// MySQL database structure
message MysqlRdbms {
  // Mysql databases on the server
  repeated MysqlDatabase mysql_databases = 1;
}

// MySQL source configuration
message MysqlSourceConfig {
  // MySQL objects to retrieve from the source.
  MysqlRdbms include_objects = 1;

  // MySQL objects to exclude from the stream.
  MysqlRdbms exclude_objects = 2;

  // Maximum number of concurrent CDC tasks. The number should be non negative.
  // If not set (or set to 0), the system's default value will be used.
  int32 max_concurrent_cdc_tasks = 3;
}

// The configuration of the stream source.
message SourceConfig {
  // Required. Source connection profile resoource.
  // Format: `projects/{project}/locations/{location}/connectionProfiles/{name}`
  string source_connection_profile = 1;

  // Stream configuration that is specific to the data source type.
  oneof source_stream_config {
    // Oracle data source configuration.
    OracleSourceConfig oracle_source_config = 100;

    // MySQL data source configuration.
    MysqlSourceConfig mysql_source_config = 101;

    // PostgreSQL data source configuration.
    PostgresqlSourceConfig postgresql_source_config = 102;
  }
}

// AVRO file format configuration.
message AvroFileFormat {

}

// JSON file format configuration.
message JsonFileFormat {
  // Schema file format.
  enum SchemaFileFormat {
    // Unspecified schema file format.
    SCHEMA_FILE_FORMAT_UNSPECIFIED = 0;

    // Do not attach schema file.
    NO_SCHEMA_FILE = 1;

    // Avro schema format.
    AVRO_SCHEMA_FILE = 2;
  }

  // Json file compression.
  enum JsonCompression {
    // Unspecified json file compression.
    JSON_COMPRESSION_UNSPECIFIED = 0;

    // Do not compress JSON file.
    NO_COMPRESSION = 1;

    // Gzip compression.
    GZIP = 2;
  }

  // The schema file format along JSON data files.
  SchemaFileFormat schema_file_format = 1;

  // Compression of the loaded JSON file.
  JsonCompression compression = 2;
}

// Google Cloud Storage destination configuration
message GcsDestinationConfig {
  // Path inside the Cloud Storage bucket to write data to.
  string path = 1;

  // The maximum file size to be saved in the bucket.
  int32 file_rotation_mb = 2;

  // The maximum duration for which new events are added before a file is
  // closed and a new file is created.
  google.protobuf.Duration file_rotation_interval = 3;

  // File Format that the data should be written in.
  oneof file_format {
    // AVRO file format configuration.
    AvroFileFormat avro_file_format = 100;

    // JSON file format configuration.
    JsonFileFormat json_file_format = 101;
  }
}

message BigQueryDestinationConfig {
  // A single target dataset to which all data will be streamed.
  message SingleTargetDataset {
    // The dataset ID of the target dataset.
    string dataset_id = 1;
  }

  // Destination datasets are created so that hierarchy of the destination data
  // objects matches the source hierarchy.
  message SourceHierarchyDatasets {
    // Dataset template used for dynamic dataset creation.
    message DatasetTemplate {
      // Required. The geographic location where the dataset should reside. See
      // https://cloud.google.com/bigquery/docs/locations for supported
      // locations.
      string location = 1;

      // If supplied, every created dataset will have its name prefixed by the
      // provided value. The prefix and name will be separated by an underscore.
      // i.e. <prefix>_<dataset_name>.
      string dataset_id_prefix = 2;

      // Describes the Cloud KMS encryption key that will be used to
      // protect destination BigQuery table. The BigQuery Service Account
      // associated with your project requires access to this encryption key.
      // i.e.
      // projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}.
      // See https://cloud.google.com/bigquery/docs/customer-managed-encryption
      // for more information.
      string kms_key_name = 3;
    }

    DatasetTemplate dataset_template = 2;
  }

  // Target dataset(s) configuration.
  oneof dataset_config {
    // Single destination dataset.
    SingleTargetDataset single_target_dataset = 201;

    // Source hierarchy datasets.
    SourceHierarchyDatasets source_hierarchy_datasets = 202;
  }

  // The guaranteed data freshness (in seconds) when querying tables created by
  // the stream. Editing this field will only affect new tables created in the
  // future, but existing tables will not be impacted. Lower values mean that
  // queries will return fresher data, but may result in higher cost.
  google.protobuf.Duration data_freshness = 300;
}

// The configuration of the stream destination.
message DestinationConfig {
  // Required. Destination connection profile resource.
  // Format: `projects/{project}/locations/{location}/connectionProfiles/{name}`
  string destination_connection_profile = 1;

  // Stream configuration that is specific to the data destination type.
  oneof destination_stream_config {
    // A configuration for how data should be loaded to Cloud Storage.
    GcsDestinationConfig gcs_destination_config = 100;

    // BigQuery destination configuration.
    BigQueryDestinationConfig bigquery_destination_config = 101;
  }
}

// A resource representing streaming data from a source to a destination.
message Stream {
  // Backfill strategy to automatically backfill the Stream's objects.
  // Specific objects can be excluded.
  message BackfillAllStrategy {
    // List of objects to exclude.
    oneof excluded_objects {
      // Oracle data source objects to avoid backfilling.
      OracleRdbms oracle_excluded_objects = 1;

      // MySQL data source objects to avoid backfilling.
      MysqlRdbms mysql_excluded_objects = 2;

      // PostgreSQL data source objects to avoid backfilling.
      PostgresqlRdbms postgresql_excluded_objects = 3;
    }
  }

  // Backfill strategy to disable automatic backfill for the Stream's objects.
  message BackfillNoneStrategy {

  }

  // Stream state.
  enum State {
    // Unspecified stream state.
    STATE_UNSPECIFIED = 0;

    // The stream has been created but has not yet started streaming data.
    NOT_STARTED = 1;

    // The stream is running.
    RUNNING = 2;

    // The stream is paused.
    PAUSED = 3;

    // The stream is in maintenance mode.
    //
    // Updates are rejected on the resource in this state.
    MAINTENANCE = 4;

    // The stream is experiencing an error that is preventing data from being
    // streamed.
    FAILED = 5;

    // The stream has experienced a terminal failure.
    FAILED_PERMANENTLY = 6;

    // The stream is starting, but not yet running.
    STARTING = 7;

    // The Stream is no longer reading new events, but still writing events in
    // the buffer.
    DRAINING = 8;
  }

  // Output only. The stream's name.
  string name = 1;

  // Output only. The creation time of the stream.
  google.protobuf.Timestamp create_time = 2;

  // Output only. The last update time of the stream.
  google.protobuf.Timestamp update_time = 3;

  // Labels.
  map<string, string> labels = 4;

  // Required. Display name.
  string display_name = 5;

  // Required. Source connection profile configuration.
  SourceConfig source_config = 6;

  // Required. Destination connection profile configuration.
  DestinationConfig destination_config = 7;

  // The state of the stream.
  State state = 8;

  // Stream backfill strategy.
  oneof backfill_strategy {
    // Automatically backfill objects included in the stream source
    // configuration. Specific objects can be excluded.
    BackfillAllStrategy backfill_all = 101;

    // Do not automatically backfill any objects.
    BackfillNoneStrategy backfill_none = 102;
  }

  // Output only. Errors on the Stream.
  repeated Error errors = 9;

  // Immutable. A reference to a KMS encryption key.
  // If provided, it will be used to encrypt the data.
  // If left blank, data will be encrypted using an internal Stream-specific
  // encryption key provisioned through KMS.
  optional string customer_managed_encryption_key = 10;
}

// Represent a user-facing Error.
message Error {
  // A title that explains the reason for the error.
  string reason = 1;

  // A unique identifier for this specific error,
  // allowing it to be traced throughout the system in logs and API responses.
  string error_uuid = 2;

  // A message containing more information about the error that occurred.
  string message = 3;

  // The time when the error occurred.
  google.protobuf.Timestamp error_time = 4;

  // Additional information about the error.
  map<string, string> details = 5;
}

// The data within all ConnectionProfile events.
message ConnectionProfileEventData {
  // Optional. The ConnectionProfile event payload. Unset for deletion events.
  optional ConnectionProfile payload = 1;
}

// The data within all PrivateConnection events.
message PrivateConnectionEventData {
  // Optional. The PrivateConnection event payload. Unset for deletion events.
  optional PrivateConnection payload = 1;
}

// The data within all Stream events.
message StreamEventData {
  // Optional. The Stream event payload. Unset for deletion events.
  optional Stream payload = 1;
}

// The data within all Route events.
message RouteEventData {
  // Optional. The Route event payload. Unset for deletion events.
  optional Route payload = 1;
}