STAC-20844: optional traces table creation

StackVista · Feb 21, 2024 · 2f542ea · 2f542ea
1 parent 7dac9e4
commit 2f542ea
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 57 deletions.
diff --git a/exporter/clickhousestsexporter/README.md b/exporter/clickhousestsexporter/README.md
@@ -289,6 +289,7 @@ ClickHouse tables:
 - `logs_table_name` (default = otel_logs): The table name for logs.
 - `traces_table_name` (default = otel_traces): The table name for traces.
 - `metrics_table_name` (default = otel_metrics): The table name for metrics.
+- `create_traces_table` (default = true): Create the traces table on startup
 
 Processing:
 
@@ -329,6 +330,7 @@ exporters:
     logs_table_name: otel_logs
     traces_table_name: otel_traces
     metrics_table_name: otel_metrics
+    create_traces_table: true
     timeout: 5s
     retry_on_failure:
       enabled: true

diff --git a/exporter/clickhousestsexporter/config.go b/exporter/clickhousestsexporter/config.go
@@ -43,6 +43,8 @@ type Config struct {
 	TTLDays uint `mapstructure:"ttl_days"`
 	// TTL is The data time-to-live example 30m, 48h. 0 means no ttl.
 	TTL time.Duration `mapstructure:"ttl"`
+	// Create the traces table on startup
+	CreateTracesTable bool `mapstructure:"create_traces_table"`
 }
 
 const defaultDatabase = "default"

diff --git a/exporter/clickhousestsexporter/example/otel-collector-config.yml b/exporter/clickhousestsexporter/example/otel-collector-config.yml
@@ -17,6 +17,7 @@ exporters:
     database: otel
     logs_table_name: otel_logs
     traces_table_name: otel_traces
+    create_traces_table: true
     ttl: 12h
     timeout: 10s
     sending_queue:

diff --git a/exporter/clickhousestsexporter/exporter_traces.go b/exporter/clickhousestsexporter/exporter_traces.go
@@ -40,6 +40,10 @@ func newTracesExporter(logger *zap.Logger, cfg *Config) (*tracesExporter, error)
 }
 
 func (e *tracesExporter) start(ctx context.Context, _ component.Host) error {
+	if !e.cfg.CreateTracesTable {
+		return nil
+	}
+
 	if err := createDatabase(ctx, e.cfg); err != nil {
 		return err
 	}
@@ -200,12 +204,6 @@ CREATE TABLE IF NOT EXISTS %s (
          TraceState String,
          Attributes Map(LowCardinality(String), String)
      ) CODEC(ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE MergeTree()
 %s
 PARTITION BY toDate(Timestamp)
@@ -264,42 +262,10 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
                                   )`
 )
 
-const (
-	createTraceIDTsTableSQL = `
-create table IF NOT EXISTS %s_trace_id_ts (
-     TraceId String CODEC(ZSTD(1)),
-     Start DateTime64(9) CODEC(Delta, ZSTD(1)),
-     End DateTime64(9) CODEC(Delta, ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
-) ENGINE MergeTree()
-%s
-ORDER BY (TraceId, toUnixTimestamp(Start))
-SETTINGS index_granularity=8192;
-`
-	createTraceIDTsMaterializedViewSQL = `
-CREATE MATERIALIZED VIEW IF NOT EXISTS %s_trace_id_ts_mv
-TO %s.%s_trace_id_ts
-AS SELECT
-TraceId,
-min(Timestamp) as Start,
-max(Timestamp) as End
-FROM
-%s.%s
-WHERE TraceId!=''
-GROUP BY TraceId;
-`
-)
-
 func createTracesTable(ctx context.Context, cfg *Config, db *sql.DB) error {
 	if _, err := db.ExecContext(ctx, renderCreateTracesTableSQL(cfg)); err != nil {
 		return fmt.Errorf("exec create traces table sql: %w", err)
 	}
-	if _, err := db.ExecContext(ctx, renderCreateTraceIDTsTableSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs table sql: %w", err)
-	}
-	if _, err := db.ExecContext(ctx, renderTraceIDTsMaterializedViewSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs view sql: %w", err)
-	}
 	return nil
 }
 
@@ -311,13 +277,3 @@ func renderCreateTracesTableSQL(cfg *Config) string {
 	ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "Timestamp")
 	return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, ttlExpr)
 }
-
-func renderCreateTraceIDTsTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "Start")
-	return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, ttlExpr)
-}
-
-func renderTraceIDTsMaterializedViewSQL(cfg *Config) string {
-	return fmt.Sprintf(createTraceIDTsMaterializedViewSQL, cfg.TracesTableName,
-		cfg.Database, cfg.TracesTableName, cfg.Database, cfg.TracesTableName)
-}
diff --git a/exporter/clickhousestsexporter/factory.go b/exporter/clickhousestsexporter/factory.go
@@ -34,15 +34,16 @@ func createDefaultConfig() component.Config {
 	queueSettings.NumConsumers = 1
 
 	return &Config{
-		TimeoutSettings:  exporterhelper.NewDefaultTimeoutSettings(),
-		QueueSettings:    queueSettings,
-		BackOffConfig:    configretry.NewDefaultBackOffConfig(),
-		ConnectionParams: map[string]string{},
-		Database:         defaultDatabase,
-		LogsTableName:    "otel_logs",
-		TracesTableName:  "otel_traces",
-		MetricsTableName: "otel_metrics",
-		TTL:              0,
+		TimeoutSettings:   exporterhelper.NewDefaultTimeoutSettings(),
+		QueueSettings:     queueSettings,
+		BackOffConfig:     configretry.NewDefaultBackOffConfig(),
+		ConnectionParams:  map[string]string{},
+		Database:          defaultDatabase,
+		LogsTableName:     "otel_logs",
+		TracesTableName:   "otel_traces",
+		MetricsTableName:  "otel_metrics",
+		CreateTracesTable: true,
+		TTL:               0,
 	}
 }
 

diff --git a/exporter/clickhousestsexporter/testdata/config.yaml b/exporter/clickhousestsexporter/testdata/config.yaml
@@ -8,6 +8,7 @@ clickhousests/full:
   ttl: 72h
   logs_table_name: otel_logs
   traces_table_name: otel_traces
+  create_traces_table: true
   timeout: 5s
   retry_on_failure:
     enabled: true