From f07caf769e6e6316a43495119f0b021f201e7f4a Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Tue, 6 Oct 2026 22:08:06 -0700 Subject: [PATCH 1/4] feat: replicated schema migration --- .changeset/replicated-engine.md | 19 ++ docker-compose.dev.yml | 7 + docker/otel-collector/entrypoint.sh | 10 +- .../schema/seed/00001_create_database.sql | 2 +- .../schema/seed/00002_otel_logs.sql | 2 +- .../schema/seed/00002_otel_logs_compat.sql | 2 +- .../schema/seed/00003_otel_metrics.sql | 10 +- .../schema/seed/00004_hyperdx_sessions.sql | 2 +- .../schema/seed/00005_otel_traces.sql | 2 +- .../schema/seed/00005_otel_traces_compat.sql | 2 +- .../schema/seed/00006_otel_logs_rollups.sql | 2 +- .../schema/seed/00007_otel_traces_rollups.sql | 2 +- packages/otel-collector/cmd/migrate/main.go | 151 ++++++++++- .../otel-collector/cmd/migrate/main_test.go | 234 +++++++++++++++++- 14 files changed, 421 insertions(+), 26 deletions(-) create mode 100644 .changeset/replicated-engine.md diff --git a/.changeset/replicated-engine.md b/.changeset/replicated-engine.md new file mode 100644 index 0000000000..fe38cf1591 --- /dev/null +++ b/.changeset/replicated-engine.md @@ -0,0 +1,19 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: opt-in Replicated database and table engines for the schema seed + +When `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED=true`, the migrate tool +creates the target database with the `Replicated` engine and seeds +`ReplicatedMergeTree` / `ReplicatedSummingMergeTree` tables instead of +`MergeTree` / `SummingMergeTree`. If the database already exists with another +engine, it logs a warning and seeds the plain engines as before. Off by default. + +The migrate tool also warns when it targets a database other than `default` +while `default` still holds ClickStack tables, since those are not migrated. + +With `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` set (off by default), +a target database that does not exist yet yields to the fallback database when +the fallback already has ClickStack tables. The collector then writes to the +fallback database too. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9c3c0491ec..572c9cbcf3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -44,6 +44,13 @@ services: # HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL # Apply TTL changes to already-existing tables too (default false): # HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true' + # Create the database with the Replicated engine and seed Replicated* table + # engines (default false). The database must not exist yet (or already be + # Replicated) and the server needs {shard}/{replica} macros: + # HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: 'true' + # Keep using this database instead when HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + # does not exist yet but this one already has ClickStack tables (default off): + # HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: 'default' ENABLE_PROMQL: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index 8995f71c06..00a490f959 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -34,7 +34,15 @@ if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then # - CLICKHOUSE_TLS_SERVER_NAME_OVERRIDE: Server name for TLS verification # - CLICKHOUSE_TLS_INSECURE_SKIP_VERIFY: Skip TLS verification (set to "true") echo "🚀 Using Go-based migrate tool with TLS support 🔐" - migrate /etc/otel/schema/seed + # With HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE set, migrate records + # the database it seeded here so the collector writes to the same one. + RESOLVED_DATABASE_PATH="${HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH:-/tmp/hyperdx-otel-exporter-database}" + rm -f "$RESOLVED_DATABASE_PATH" + HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH="$RESOLVED_DATABASE_PATH" migrate /etc/otel/schema/seed + if [ -s "$RESOLVED_DATABASE_PATH" ]; then + export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE="$(cat "$RESOLVED_DATABASE_PATH")" + echo "Using ClickHouse database from migrate: $HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE" + fi fi # Check if OPAMP_SERVER_URL is defined to determine mode diff --git a/docker/otel-collector/schema/seed/00001_create_database.sql b/docker/otel-collector/schema/seed/00001_create_database.sql index 2199bf8e40..cc4f5aa75e 100644 --- a/docker/otel-collector/schema/seed/00001_create_database.sql +++ b/docker/otel-collector/schema/seed/00001_create_database.sql @@ -1,3 +1,3 @@ -- +goose Up -- +goose NO TRANSACTION -CREATE DATABASE IF NOT EXISTS ${DATABASE}; +CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE}; diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 2664576ec8..8a875afc82 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql index 2aa6f1c82f..3c82c9a858 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00003_otel_metrics.sql b/docker/otel-collector/schema/seed/00003_otel_metrics.sql index 03938e811a..e6e90923d0 100644 --- a/docker/otel-collector/schema/seed/00003_otel_metrics.sql +++ b/docker/otel-collector/schema/seed/00003_otel_metrics.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -164,7 +164,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -201,7 +201,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index d82bccfd55..3e85bb8bf2 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) ORDER BY (ServiceName, TimestampTime, Timestamp) diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index ce964efb37..58e6de69f1 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + ${TRACES_TTL} diff --git a/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql b/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql index 480be6b093..3e3ebdfce4 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + ${TRACES_TTL} diff --git a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql index 82db4d1be6..6509663795 100644 --- a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql +++ b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 ) -ENGINE = SummingMergeTree +ENGINE = ${ENGINE_PREFIX}SummingMergeTree PARTITION BY toDate(Timestamp) ORDER BY (ColumnIdentifier, Key, Timestamp, Value) TTL Timestamp + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql index 80afc569cf..07ed18e573 100644 --- a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql +++ b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 ) -ENGINE = SummingMergeTree +ENGINE = ${ENGINE_PREFIX}SummingMergeTree PARTITION BY toDate(Timestamp) ORDER BY (ColumnIdentifier, Key, Timestamp, Value) TTL Timestamp + ${TRACES_TTL} diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index fc4e514ed0..c525c5e715 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "errors" "fmt" "log" "net/url" @@ -42,6 +43,17 @@ type Config struct { // Reconcile TTL on already-existing tables, not just newly created ones. ReconcileTableTTL bool + // Create the database with the Replicated engine and seed Replicated* + // table engines, unless the database already exists with another engine. + Replicated bool + + // Database to stay on when Database does not exist yet but this one already + // holds ClickStack tables. Empty disables the fallback. + FallbackDatabase string + + // Where to record the seeded database for the entrypoint (fallback only). + ResolvedDatabasePath string + // TLS settings TLSCAFile string TLSCertFile string @@ -68,6 +80,9 @@ func loadConfig() (*Config, error) { MetricsTTL: getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL), SessionsTTL: getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL), ReconcileTableTTL: getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true", + Replicated: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED", "false") == "true", + FallbackDatabase: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", ""), + ResolvedDatabasePath: getEnv("HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH", "/tmp/hyperdx-otel-exporter-database"), TLSCAFile: getEnv("CLICKHOUSE_TLS_CA_FILE", ""), TLSCertFile: getEnv("CLICKHOUSE_TLS_CERT_FILE", ""), TLSKeyFile: getEnv("CLICKHOUSE_TLS_KEY_FILE", ""), @@ -528,14 +543,23 @@ func reconcileTableTTLs(ctx context.Context, db *sql.DB, database string, ttlExp } // processSchemaDir creates a temporary directory with SQL files that have the -// ${DATABASE} and per-signal ${*_TTL} macros replaced with actual values. +// ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and per-signal ${*_TTL} +// macros replaced with actual values. When replicated is set, the database is +// created with the Replicated engine and MergeTree becomes ReplicatedMergeTree; +// otherwise both macros are empty. // ttlExprs maps a macro name (e.g. "LOGS_TTL") to its ClickHouse interval expr. -func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (string, error) { +func processSchemaDir(schemaDir, database string, replicated bool, ttlExprs map[string]string) (string, error) { tempDir, err := os.MkdirTemp("", "schema-*") if err != nil { return "", fmt.Errorf("failed to create temp directory: %w", err) } + databaseEngine, enginePrefix := "", "" + if replicated { + databaseEngine = fmt.Sprintf(" ENGINE = Replicated('/clickhouse/databases/%s', '{shard}', '{replica}')", database) + enginePrefix = "Replicated" + } + // Walk through the schema directory and process SQL files err = filepath.Walk(schemaDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -562,6 +586,8 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s // Replace macros with actual values processedContent := strings.ReplaceAll(string(content), "${DATABASE}", database) + processedContent = strings.ReplaceAll(processedContent, "${DATABASE_ENGINE}", databaseEngine) + processedContent = strings.ReplaceAll(processedContent, "${ENGINE_PREFIX}", enginePrefix) for macro, expr := range ttlExprs { processedContent = strings.ReplaceAll(processedContent, "${"+macro+"}", expr) } @@ -582,6 +608,75 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s return tempDir, nil } +// planReplicated decides whether to seed Replicated engines. Argument-less +// Replicated* table engines only work inside a Replicated database and an +// existing database cannot change engine, so only a missing or already +// Replicated database qualifies. Returns a warning when the flag cannot apply. +func planReplicated(enabled, exists bool, database, engine string) (bool, string) { + switch { + case !enabled: + return false, "" + case !exists, engine == "Replicated": + return true, "" + default: + return false, fmt.Sprintf("database %s already exists with engine %s, so tables use non-replicated engines; "+ + "set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE to a new database name to seed a Replicated database", database, engine) + } +} + +// getDatabaseEngine returns the engine of a database, or exists=false if it is missing. +func getDatabaseEngine(ctx context.Context, db *sql.DB, database string) (engine string, exists bool, err error) { + err = db.QueryRowContext(ctx, "SELECT engine FROM system.databases WHERE name = ?", database).Scan(&engine) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("failed to query engine of database %s: %w", database, err) + } + return engine, true, nil +} + +// legacyDatabase is where an earlier install keeps its ClickStack tables: the +// fallback database when set, otherwise default. +func legacyDatabase(fallback string) string { + if fallback != "" { + return fallback + } + return "default" +} + +// hasClickStackTables reports whether database already holds ClickStack tables. +func hasClickStackTables(ctx context.Context, db *sql.DB, database string) (bool, error) { + var n uint64 + err := db.QueryRowContext(ctx, + "SELECT count() FROM system.tables WHERE database = ? AND name IN ('otel_logs', 'otel_traces')", database).Scan(&n) + if err != nil { + return false, fmt.Errorf("failed to check database %s for ClickStack tables: %w", database, err) + } + return n > 0, nil +} + +// resolveDatabase picks the database to seed and hand to the collector. With a +// fallback set, a target that does not exist yet yields to a fallback that +// already holds ClickStack tables, so an upgrade that switched databases keeps +// writing where HyperDX sources read. Otherwise legacy tables only warrant a +// warning, as they are not migrated. +func resolveDatabase(target, fallback string, targetExists, legacyHasTables bool) (string, string) { + legacy := legacyDatabase(fallback) + switch { + case legacy == target || !legacyHasTables: + return target, "" + case fallback != "" && !targetExists: + return fallback, fmt.Sprintf("database %s does not exist and %s already has ClickStack tables, so staying on %s. "+ + "Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to silence this, or clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to %s", + target, fallback, fallback, fallback, target) + default: + return target, fmt.Sprintf("database %s has existing ClickStack tables that are not migrated to %s; "+ + "HyperDX sources created earlier still point at %s. Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to keep using the old database", + legacy, target, legacy, legacy) + } +} + // runMigrationWithRetry runs goose seed with exponential backoff retry func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string, maxRetries int) error { var lastErr error @@ -788,9 +883,41 @@ func main() { log.Printf("Table TTLs: logs=%s traces=%s metrics=%s sessions=%s (default %s)", cfg.LogsTTL, cfg.TracesTTL, cfg.MetricsTTL, cfg.SessionsTTL, cfg.TablesTTL) - // Process schema directory (replace ${DATABASE} and ${*_TTL} macros) - log.Printf("Preparing SQL files with database: %s", cfg.Database) - tempDir, err := processSchemaDir(cfg.SchemaDir, cfg.Database, ttlExprs) + // The legacy tables check is best effort unless the fallback depends on it. + var legacyHasTables bool + if legacy := legacyDatabase(cfg.FallbackDatabase); legacy != cfg.Database { + if legacyHasTables, err = hasClickStackTables(ctx, db, legacy); err != nil && cfg.FallbackDatabase != "" { + log.Fatalf("Failed to check fallback database: %v", err) + } + } + + var dbEngine string + var dbExists bool + if cfg.Replicated || cfg.FallbackDatabase != "" { + if dbEngine, dbExists, err = getDatabaseEngine(ctx, db, cfg.Database); err != nil { + log.Fatalf("Failed to check database engine: %v", err) + } + } + database, warning := resolveDatabase(cfg.Database, cfg.FallbackDatabase, dbExists, legacyHasTables) + if warning != "" { + log.Printf("WARNING: %s", warning) + } + + // A fallback database keeps the plain engines it was seeded with. + replicated := false + if database == cfg.Database { + replicated, warning = planReplicated(cfg.Replicated, dbExists, database, dbEngine) + if warning != "" { + log.Printf("WARNING: %s", warning) + } + if replicated { + log.Printf("Using Replicated database and table engines for %s", database) + } + } + + // Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and ${*_TTL} macros) + log.Printf("Preparing SQL files with database: %s", database) + tempDir, err := processSchemaDir(cfg.SchemaDir, database, replicated, ttlExprs) if err != nil { log.Fatalf("Failed to process schema directory: %v", err) } @@ -841,11 +968,23 @@ func main() { // CREATE TABLE IF NOT EXISTS won't update TTL on existing tables; reconcile // (opt-in) applies it to them too. Non-fatal so it can't block startup. if cfg.ReconcileTableTTL { - if err := reconcileTableTTLs(ctx, db, cfg.Database, ttlExprs); err != nil { + if err := reconcileTableTTLs(ctx, db, database, ttlExprs); err != nil { log.Printf("WARNING: reconcile: %v", err) } } + // Record the seeded database so the entrypoint hands it to the collector. + if cfg.FallbackDatabase != "" { + if err := os.WriteFile(cfg.ResolvedDatabasePath, []byte(database+"\n"), 0644); err != nil { + // The collector would otherwise write to a database that was not seeded. + if database != cfg.Database { + log.Fatalf("Failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err) + } + + log.Printf("WARNING: failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err) + } + } + log.Println("========================================") log.Println("Schema seed completed successfully") log.Println("========================================") diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index ba066ef6b8..a7ba097496 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -10,6 +10,7 @@ import ( "math/big" "os" "path/filepath" + "regexp" "strings" "testing" "time" @@ -557,7 +558,7 @@ func TestProcessSchemaDir_Subdirectories(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "testdb", map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) + tempDir, err := processSchemaDir(schemaDir, "testdb", false, map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) if err != nil { t.Fatal(err) } @@ -580,7 +581,7 @@ func TestProcessSchemaDir_MultipleReplacements(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) if err != nil { t.Fatal(err) } @@ -603,7 +604,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) } @@ -619,7 +620,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { } func TestProcessSchemaDir_NonexistentDir(t *testing.T) { - _, err := processSchemaDir("/nonexistent/schema", "db", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + _, err := processSchemaDir("/nonexistent/schema", "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err == nil { t.Fatal("expected error for nonexistent schema dir") } @@ -804,7 +805,7 @@ func TestProcessSchemaDir_PerSignalMacros(t *testing.T) { if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{ + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{ "LOGS_TTL": "toIntervalDay(180)", "TRACES_TTL": "toIntervalDay(180)", "METRICS_TTL": "toIntervalDay(30)", @@ -824,6 +825,227 @@ func TestProcessSchemaDir_PerSignalMacros(t *testing.T) { } } +// --------------------------------------------------------------------------- +// replicated engines +// --------------------------------------------------------------------------- + +func TestLoadConfig_Replicated(t *testing.T) { + origArgs := os.Args + t.Cleanup(func() { os.Args = origArgs }) + os.Args = []string{"migrate", t.TempDir()} + + for _, c := range []struct { + value string + want bool + }{{"", false}, {"1", false}, {"TRUE", false}, {"true", true}} { + t.Setenv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED", c.value) + cfg, err := loadConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Replicated != c.want { + t.Errorf("REPLICATED=%q: got %v, want %v", c.value, cfg.Replicated, c.want) + } + } +} + +func TestProcessSchemaDir_ReplicatedMacros(t *testing.T) { + schemaDir := t.TempDir() + content := "CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE};\n" + + "ENGINE = ${ENGINE_PREFIX}MergeTree; ENGINE = ${ENGINE_PREFIX}SummingMergeTree; ENGINE = TimeSeries" + if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + replicated bool + want string + }{ + {false, "CREATE DATABASE IF NOT EXISTS otel;\n" + + "ENGINE = MergeTree; ENGINE = SummingMergeTree; ENGINE = TimeSeries"}, + {true, "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + + "ENGINE = ReplicatedMergeTree; ENGINE = ReplicatedSummingMergeTree; ENGINE = TimeSeries"}, + } { + tempDir, err := processSchemaDir(schemaDir, "otel", c.replicated, map[string]string{}) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + got, err := os.ReadFile(filepath.Join(tempDir, "001.sql")) + if err != nil { + t.Fatal(err) + } + if string(got) != c.want { + t.Errorf("replicated=%v: got %q, want %q", c.replicated, string(got), c.want) + } + } +} + +// Guards the real seed files: every MergeTree-family engine must use the +// ${ENGINE_PREFIX} macro and take no arguments (a Replicated database rejects +// explicit Keeper path/replica arguments), and the database must be created +// with the Replicated engine only when enabled. +func TestSeedSchemaEngines(t *testing.T) { + seedDir := filepath.Join("..", "..", "..", "..", "docker", "otel-collector", "schema", "seed") + ttlExprs := map[string]string{} + for _, m := range []string{"TABLES_TTL", "LOGS_TTL", "TRACES_TTL", "METRICS_TTL", "SESSIONS_TTL"} { + ttlExprs[m] = "toIntervalDay(30)" + } + engineRe := regexp.MustCompile(`(?im)^\s*ENGINE\s*=\s*(.*)$`) + for _, replicated := range []bool{false, true} { + tempDir, err := processSchemaDir(seedDir, "otel", replicated, ttlExprs) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + files, err := listSQLFiles(tempDir) + if err != nil { + t.Fatal(err) + } + mergeTrees := 0 + for _, f := range files { + content, err := os.ReadFile(filepath.Join(tempDir, f)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "${") { + t.Errorf("replicated=%v: %s has an unreplaced macro", replicated, f) + } + for _, m := range engineRe.FindAllStringSubmatch(string(content), -1) { + engine := strings.TrimSpace(m[1]) + if engine == "TimeSeries" { + continue + } + mergeTrees++ + if strings.Contains(engine, "(") { + t.Errorf("%s: engine %q must not take arguments", f, engine) + } + if strings.HasPrefix(engine, "Replicated") != replicated || !strings.HasSuffix(engine, "MergeTree") { + t.Errorf("replicated=%v: %s has engine %q", replicated, f, engine) + } + } + } + if mergeTrees == 0 { + t.Errorf("replicated=%v: no MergeTree engines found in %s", replicated, seedDir) + } + + createDB, err := os.ReadFile(filepath.Join(tempDir, "00001_create_database.sql")) + if err != nil { + t.Fatal(err) + } + want := "CREATE DATABASE IF NOT EXISTS otel;\n" + if replicated { + want = "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + } + if !strings.HasSuffix(string(createDB), want) { + t.Errorf("replicated=%v: 00001_create_database.sql = %q; want suffix %q", replicated, string(createDB), want) + } + } +} + +func TestPlanReplicated(t *testing.T) { + cases := []struct { + name string + enabled, exists bool + engine string + want, warns bool + }{ + {"flag off", false, true, "Atomic", false, false}, + {"flag off, database missing", false, false, "", false, false}, + {"flag off, database Replicated", false, true, "Replicated", false, false}, + {"database missing", true, false, "", true, false}, + {"database Replicated", true, true, "Replicated", true, false}, + {"database Atomic", true, true, "Atomic", false, true}, + {"database Shared", true, true, "Shared", false, true}, + } + for _, c := range cases { + got, warning := planReplicated(c.enabled, c.exists, "otel", c.engine) + if got != c.want { + t.Errorf("%s: replicated = %v; want %v", c.name, got, c.want) + } + if (warning != "") != c.warns { + t.Errorf("%s: warning = %q; want warning=%v", c.name, warning, c.warns) + } + if c.warns && (!strings.Contains(warning, c.engine) || !strings.Contains(warning, "otel")) { + t.Errorf("%s: warning should name the database and engine: %q", c.name, warning) + } + } +} + +// --------------------------------------------------------------------------- +// fallback database +// --------------------------------------------------------------------------- + +func TestLoadConfig_FallbackDatabase(t *testing.T) { + origArgs := os.Args + t.Cleanup(func() { os.Args = origArgs }) + os.Args = []string{"migrate", t.TempDir()} + + cfg, err := loadConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.FallbackDatabase != "" { + t.Errorf("FallbackDatabase should default to empty, got %q", cfg.FallbackDatabase) + } + if cfg.ResolvedDatabasePath != "/tmp/hyperdx-otel-exporter-database" { + t.Errorf("ResolvedDatabasePath: got %q", cfg.ResolvedDatabasePath) + } + + t.Setenv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", "default") + t.Setenv("HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH", "/run/hdx/database") + if cfg, err = loadConfig(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.FallbackDatabase != "default" || cfg.ResolvedDatabasePath != "/run/hdx/database" { + t.Errorf("overrides not applied: fallback=%q path=%q", cfg.FallbackDatabase, cfg.ResolvedDatabasePath) + } +} + +func TestResolveDatabase(t *testing.T) { + cases := []struct { + name string + target, fallback string + targetExists, legacyHasTables bool + want string + wantWarning []string // substrings; nil means no warning + }{ + {"no fallback, target default", "default", "", true, true, "default", nil}, + {"no fallback, legacy tables in default", "otel", "", false, true, "otel", + []string{"database default has existing ClickStack tables", "not migrated to otel", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default"}}, + {"no fallback, default empty", "otel", "", false, false, "otel", nil}, + {"fallback, target missing, fallback seeded", "otel", "default", false, true, "default", + []string{"database otel does not exist", "staying on default", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default", "clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to otel"}}, + {"fallback, target exists, fallback seeded", "otel", "default", true, true, "otel", + []string{"database default has existing ClickStack tables", "not migrated to otel"}}, + {"fallback, target missing, fallback empty", "otel", "default", false, false, "otel", nil}, + {"fallback equals target", "default", "default", true, true, "default", nil}, + {"custom fallback", "otel", "legacy", false, true, "legacy", []string{"staying on legacy"}}, + } + for _, c := range cases { + got, warning := resolveDatabase(c.target, c.fallback, c.targetExists, c.legacyHasTables) + if got != c.want { + t.Errorf("%s: database = %q; want %q", c.name, got, c.want) + } + if c.wantWarning == nil && warning != "" { + t.Errorf("%s: unexpected warning %q", c.name, warning) + } + for _, sub := range c.wantWarning { + if !strings.Contains(warning, sub) { + t.Errorf("%s: warning %q should contain %q", c.name, warning, sub) + } + } + } +} + +func TestLegacyDatabase(t *testing.T) { + if got := legacyDatabase(""); got != "default" { + t.Errorf("legacyDatabase(\"\") = %q; want default", got) + } + if got := legacyDatabase("legacy"); got != "legacy" { + t.Errorf("legacyDatabase(\"legacy\") = %q; want legacy", got) + } +} + // --------------------------------------------------------------------------- // parseTTLDuration / ttlToClickHouseInterval (existing tests below) // --------------------------------------------------------------------------- @@ -1151,7 +1373,7 @@ SETTINGS ttl_only_drop_parts = 1;` t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "mydb", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "mydb", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) } From 5375b6af47ebfd14344e41c6d41080dedfb17467 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Fri, 20 Nov 2026 16:21:46 -0500 Subject: [PATCH 2/4] fix: warn about legacy tables only on first seed, respect fallback's engine, document per-node Replicated setup --- .changeset/replicated-engine.md | 10 ++++-- docker-compose.dev.yml | 4 ++- .../schema/seed/00001_create_database.sql | 2 ++ packages/otel-collector/cmd/migrate/main.go | 35 +++++++++++-------- .../otel-collector/cmd/migrate/main_test.go | 30 ++++++++-------- 5 files changed, 50 insertions(+), 31 deletions(-) diff --git a/.changeset/replicated-engine.md b/.changeset/replicated-engine.md index fe38cf1591..279531558f 100644 --- a/.changeset/replicated-engine.md +++ b/.changeset/replicated-engine.md @@ -10,8 +10,14 @@ creates the target database with the `Replicated` engine and seeds `MergeTree` / `SummingMergeTree`. If the database already exists with another engine, it logs a warning and seeds the plain engines as before. Off by default. -The migrate tool also warns when it targets a database other than `default` -while `default` still holds ClickStack tables, since those are not migrated. +The database is created only on the node migrate connects to. On a cluster, run +migrate against every replica, or create the `Replicated` database on each node +first (for example with `ON CLUSTER`); otherwise inserts that reach a node +without the database fail. + +The migrate tool also warns when it targets a database other than `default` that +has no ClickStack tables yet while `default` still holds them, since those are +not migrated. With `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` set (off by default), a target database that does not exist yet yields to the fallback database when diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0861e26b54..14e3483e2b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -51,7 +51,9 @@ services: # HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true' # Create the database with the Replicated engine and seed Replicated* table # engines (default false). The database must not exist yet (or already be - # Replicated) and the server needs {shard}/{replica} macros: + # Replicated) and the server needs {shard}/{replica} macros. Migrate creates + # the database only on the node it connects to, so on a cluster run it + # against every replica or create the database on each node first: # HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: 'true' # Keep using this database instead when HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE # does not exist yet but this one already has ClickStack tables (default off): diff --git a/docker/otel-collector/schema/seed/00001_create_database.sql b/docker/otel-collector/schema/seed/00001_create_database.sql index cc4f5aa75e..b9e51879ce 100644 --- a/docker/otel-collector/schema/seed/00001_create_database.sql +++ b/docker/otel-collector/schema/seed/00001_create_database.sql @@ -1,3 +1,5 @@ -- +goose Up -- +goose NO TRANSACTION +-- The engine macro is empty unless HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED +-- applies, then it is " ENGINE = Replicated(...)" (see processSchemaDir). CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE}; diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 9ad726a047..3b2985f107 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -664,11 +664,12 @@ func hasClickStackTables(ctx context.Context, db *sql.DB, database string) (bool // fallback set, a target that does not exist yet yields to a fallback that // already holds ClickStack tables, so an upgrade that switched databases keeps // writing where HyperDX sources read. Otherwise legacy tables only warrant a -// warning, as they are not migrated. -func resolveDatabase(target, fallback string, targetExists, legacyHasTables bool) (string, string) { +// warning, as they are not migrated, and only until the target holds +// ClickStack tables of its own: after that the move was already made. +func resolveDatabase(target, fallback string, targetExists, targetHasTables, legacyHasTables bool) (string, string) { legacy := legacyDatabase(fallback) switch { - case legacy == target || !legacyHasTables: + case legacy == target || !legacyHasTables || targetHasTables: return target, "" case fallback != "" && !targetExists: return fallback, fmt.Sprintf("database %s does not exist and %s already has ClickStack tables, so staying on %s. "+ @@ -962,11 +963,15 @@ func main() { cfg.LogsTTL, cfg.TracesTTL, cfg.MetricsTTL, cfg.SessionsTTL, cfg.TablesTTL) // The legacy tables check is best effort unless the fallback depends on it. - var legacyHasTables bool + // A failed target check only means the legacy warning may still show. + var legacyHasTables, targetHasTables bool if legacy := legacyDatabase(cfg.FallbackDatabase); legacy != cfg.Database { if legacyHasTables, err = hasClickStackTables(ctx, db, legacy); err != nil && cfg.FallbackDatabase != "" { log.Fatalf("Failed to check fallback database: %v", err) } + if legacyHasTables { + targetHasTables, _ = hasClickStackTables(ctx, db, cfg.Database) + } } var dbEngine string @@ -976,22 +981,24 @@ func main() { log.Fatalf("Failed to check database engine: %v", err) } } - database, warning := resolveDatabase(cfg.Database, cfg.FallbackDatabase, dbExists, legacyHasTables) + database, warning := resolveDatabase(cfg.Database, cfg.FallbackDatabase, dbExists, targetHasTables, legacyHasTables) if warning != "" { log.Printf("WARNING: %s", warning) } - // A fallback database keeps the plain engines it was seeded with. - replicated := false - if database == cfg.Database { - replicated, warning = planReplicated(cfg.Replicated, dbExists, database, dbEngine) - if warning != "" { - log.Printf("WARNING: %s", warning) - } - if replicated { - log.Printf("Using Replicated database and table engines for %s", database) + // A fallback already exists, so only its own engine can make it Replicated. + if cfg.Replicated && database != cfg.Database { + if dbEngine, dbExists, err = getDatabaseEngine(ctx, db, database); err != nil { + log.Fatalf("Failed to check database engine: %v", err) } } + replicated, warning := planReplicated(cfg.Replicated, dbExists, database, dbEngine) + if warning != "" { + log.Printf("WARNING: %s", warning) + } + if replicated { + log.Printf("Using Replicated database and table engines for %s", database) + } // Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and ${*_TTL} macros) log.Printf("Preparing SQL files with database: %s", database) diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index b81ffa9c05..3afc9d1e71 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -1020,26 +1020,28 @@ func TestLoadConfig_FallbackDatabase(t *testing.T) { func TestResolveDatabase(t *testing.T) { cases := []struct { - name string - target, fallback string - targetExists, legacyHasTables bool - want string - wantWarning []string // substrings; nil means no warning + name string + target, fallback string + targetExists, targetHasTables, legacyHasTables bool + want string + wantWarning []string // substrings; nil means no warning }{ - {"no fallback, target default", "default", "", true, true, "default", nil}, - {"no fallback, legacy tables in default", "otel", "", false, true, "otel", + {"no fallback, target default", "default", "", true, true, true, "default", nil}, + {"no fallback, legacy tables in default", "otel", "", false, false, true, "otel", []string{"database default has existing ClickStack tables", "not migrated to otel", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default"}}, - {"no fallback, default empty", "otel", "", false, false, "otel", nil}, - {"fallback, target missing, fallback seeded", "otel", "default", false, true, "default", + {"no fallback, target and default both seeded", "otel", "", true, true, true, "otel", nil}, + {"no fallback, default empty", "otel", "", false, false, false, "otel", nil}, + {"fallback, target missing, fallback seeded", "otel", "default", false, false, true, "default", []string{"database otel does not exist", "staying on default", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default", "clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to otel"}}, - {"fallback, target exists, fallback seeded", "otel", "default", true, true, "otel", + {"fallback, target exists empty, fallback seeded", "otel", "default", true, false, true, "otel", []string{"database default has existing ClickStack tables", "not migrated to otel"}}, - {"fallback, target missing, fallback empty", "otel", "default", false, false, "otel", nil}, - {"fallback equals target", "default", "default", true, true, "default", nil}, - {"custom fallback", "otel", "legacy", false, true, "legacy", []string{"staying on legacy"}}, + {"fallback, target and fallback both seeded", "otel", "default", true, true, true, "otel", nil}, + {"fallback, target missing, fallback empty", "otel", "default", false, false, false, "otel", nil}, + {"fallback equals target", "default", "default", true, true, true, "default", nil}, + {"custom fallback", "otel", "legacy", false, false, true, "legacy", []string{"staying on legacy"}}, } for _, c := range cases { - got, warning := resolveDatabase(c.target, c.fallback, c.targetExists, c.legacyHasTables) + got, warning := resolveDatabase(c.target, c.fallback, c.targetExists, c.targetHasTables, c.legacyHasTables) if got != c.want { t.Errorf("%s: database = %q; want %q", c.name, got, c.want) } From cfbe5987c1679d99ce30ca2aab37efe0a9f37f68 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Fri, 20 Nov 2026 17:59:03 -0500 Subject: [PATCH 3/4] fix: replicate the PromQL metrics_ts inner tables on ClickHouse 26.10+ --- .changeset/replicated-engine.md | 2 + .../seed/00008_otel_metrics_timeseries.sql | 5 +- packages/otel-collector/cmd/migrate/main.go | 40 ++++++++--- .../otel-collector/cmd/migrate/main_test.go | 72 ++++++++++++++----- 4 files changed, 93 insertions(+), 26 deletions(-) diff --git a/.changeset/replicated-engine.md b/.changeset/replicated-engine.md index 279531558f..ff4747f6c8 100644 --- a/.changeset/replicated-engine.md +++ b/.changeset/replicated-engine.md @@ -9,6 +9,8 @@ creates the target database with the `Replicated` engine and seeds `ReplicatedMergeTree` / `ReplicatedSummingMergeTree` tables instead of `MergeTree` / `SummingMergeTree`. If the database already exists with another engine, it logs a warning and seeds the plain engines as before. Off by default. +With `ENABLE_PROMQL=true` on ClickHouse 26.10 or later, the inner tables of the +PromQL `metrics_ts` table are Replicated too; older servers keep plain ones. The database is created only on the node migrate connects to. On a cluster, run migrate against every replica, or create the `Replicated` database on each node diff --git a/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql index 48b8372b44..005632d22e 100644 --- a/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql +++ b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql @@ -1,4 +1,7 @@ -- +goose Up +-- The engines macro adds Replicated inner table engines when the seed is +-- replicated. It must follow SETTINGS: placed before it, ClickHouse ignores +-- allow_experimental_time_series_table and the CREATE fails. CREATE TABLE IF NOT EXISTS ${DATABASE}.metrics_ts ENGINE = TimeSeries -SETTINGS allow_experimental_time_series_table = 1; +SETTINGS allow_experimental_time_series_table = 1${TIMESERIES_ENGINES}; diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 3b2985f107..1108ddaaa2 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -546,23 +546,35 @@ func reconcileTableTTLs(ctx context.Context, db *sql.DB, database string, ttlExp return nil } +// replicatedTimeSeriesEngines makes every TimeSeries inner table Replicated: +// DATA covers samples and recentsamples, TAGS covers tags and timeranges, and +// METRICS covers metricfamilies. Naming an inner engine pins its family, so +// these match the ClickHouse 26.10 defaults; older servers use other layouts. +const replicatedTimeSeriesEngines = "\nDATA ENGINE = ReplicatedMergeTree" + + "\nTAGS ENGINE = ReplicatedReplacingMergeTree" + + "\nMETRICS ENGINE = ReplicatedReplacingMergeTree" + // processSchemaDir creates a temporary directory with SQL files that have the -// ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and per-signal ${*_TTL} -// macros replaced with actual values. When replicated is set, the database is -// created with the Replicated engine and MergeTree becomes ReplicatedMergeTree; -// otherwise both macros are empty. +// ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX}, ${TIMESERIES_ENGINES} and +// per-signal ${*_TTL} macros replaced with actual values. When replicated is +// set, the database is created with the Replicated engine and MergeTree +// becomes ReplicatedMergeTree; replicatedTimeSeries also makes the TimeSeries +// inner tables Replicated. Unset macros are empty. // ttlExprs maps a macro name (e.g. "LOGS_TTL") to its ClickHouse interval expr. -func processSchemaDir(schemaDir, database string, replicated bool, ttlExprs map[string]string) (string, error) { +func processSchemaDir(schemaDir, database string, replicated, replicatedTimeSeries bool, ttlExprs map[string]string) (string, error) { tempDir, err := os.MkdirTemp("", "schema-*") if err != nil { return "", fmt.Errorf("failed to create temp directory: %w", err) } - databaseEngine, enginePrefix := "", "" + databaseEngine, enginePrefix, timeSeriesEngines := "", "", "" if replicated { databaseEngine = fmt.Sprintf(" ENGINE = Replicated('/clickhouse/databases/%s', '{shard}', '{replica}')", database) enginePrefix = "Replicated" } + if replicatedTimeSeries { + timeSeriesEngines = replicatedTimeSeriesEngines + } // Walk through the schema directory and process SQL files err = filepath.Walk(schemaDir, func(path string, info os.FileInfo, err error) error { @@ -592,6 +604,7 @@ func processSchemaDir(schemaDir, database string, replicated bool, ttlExprs map[ processedContent := strings.ReplaceAll(string(content), "${DATABASE}", database) processedContent = strings.ReplaceAll(processedContent, "${DATABASE_ENGINE}", databaseEngine) processedContent = strings.ReplaceAll(processedContent, "${ENGINE_PREFIX}", enginePrefix) + processedContent = strings.ReplaceAll(processedContent, "${TIMESERIES_ENGINES}", timeSeriesEngines) for macro, expr := range ttlExprs { processedContent = strings.ReplaceAll(processedContent, "${"+macro+"}", expr) } @@ -744,6 +757,13 @@ func supportsFullTextSearch(major, minor int) bool { return major > 26 || (major == 26 && minor >= 2) } +// supportsReplicatedTimeSeries returns true if replicatedTimeSeriesEngines +// matches the server's TimeSeries inner table layout. This requires ClickHouse +// >= 26.10; 26.4, for one, drops the default ORDER BY of a named inner engine. +func supportsReplicatedTimeSeries(major, minor int) bool { + return major > 26 || (major == 26 && minor >= 10) +} + // textIndexTables are the seeded tables with text indexes. var textIndexTables = []string{"otel_logs", "otel_traces", "hyperdx_sessions"} @@ -999,10 +1019,14 @@ func main() { if replicated { log.Printf("Using Replicated database and table engines for %s", database) } + replicatedTimeSeries := replicated && supportsReplicatedTimeSeries(chMajor, chMinor) + if replicated && !replicatedTimeSeries && os.Getenv("ENABLE_PROMQL") == "true" { + log.Printf("WARNING: ClickHouse %d.%d < 26.10, so the PromQL metrics_ts inner tables are not replicated", chMajor, chMinor) + } - // Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and ${*_TTL} macros) + // Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX}, ${TIMESERIES_ENGINES} and ${*_TTL} macros) log.Printf("Preparing SQL files with database: %s", database) - tempDir, err := processSchemaDir(cfg.SchemaDir, database, replicated, ttlExprs) + tempDir, err := processSchemaDir(cfg.SchemaDir, database, replicated, replicatedTimeSeries, ttlExprs) if err != nil { log.Fatalf("Failed to process schema directory: %v", err) } diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 3afc9d1e71..56ae17bac5 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -561,7 +561,7 @@ func TestProcessSchemaDir_Subdirectories(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "testdb", false, map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) + tempDir, err := processSchemaDir(schemaDir, "testdb", false, false, map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) if err != nil { t.Fatal(err) } @@ -584,7 +584,7 @@ func TestProcessSchemaDir_MultipleReplacements(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, false, map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) if err != nil { t.Fatal(err) } @@ -607,7 +607,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) } @@ -623,7 +623,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { } func TestProcessSchemaDir_NonexistentDir(t *testing.T) { - _, err := processSchemaDir("/nonexistent/schema", "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + _, err := processSchemaDir("/nonexistent/schema", "db", false, false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err == nil { t.Fatal("expected error for nonexistent schema dir") } @@ -822,7 +822,7 @@ func TestProcessSchemaDir_PerSignalMacros(t *testing.T) { if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{ + tempDir, err := processSchemaDir(schemaDir, "db", false, false, map[string]string{ "LOGS_TTL": "toIntervalDay(180)", "TRACES_TTL": "toIntervalDay(180)", "METRICS_TTL": "toIntervalDay(30)", @@ -869,20 +869,23 @@ func TestLoadConfig_Replicated(t *testing.T) { func TestProcessSchemaDir_ReplicatedMacros(t *testing.T) { schemaDir := t.TempDir() content := "CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE};\n" + - "ENGINE = ${ENGINE_PREFIX}MergeTree; ENGINE = ${ENGINE_PREFIX}SummingMergeTree; ENGINE = TimeSeries" + "ENGINE = ${ENGINE_PREFIX}MergeTree; ENGINE = ${ENGINE_PREFIX}SummingMergeTree; ENGINE = TimeSeries${TIMESERIES_ENGINES}" if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { t.Fatal(err) } for _, c := range []struct { - replicated bool - want string + replicated, replicatedTimeSeries bool + want string }{ - {false, "CREATE DATABASE IF NOT EXISTS otel;\n" + + {false, false, "CREATE DATABASE IF NOT EXISTS otel;\n" + "ENGINE = MergeTree; ENGINE = SummingMergeTree; ENGINE = TimeSeries"}, - {true, "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + + {true, false, "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + "ENGINE = ReplicatedMergeTree; ENGINE = ReplicatedSummingMergeTree; ENGINE = TimeSeries"}, + {true, true, "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + + "ENGINE = ReplicatedMergeTree; ENGINE = ReplicatedSummingMergeTree; ENGINE = TimeSeries" + + "\nDATA ENGINE = ReplicatedMergeTree\nTAGS ENGINE = ReplicatedReplacingMergeTree\nMETRICS ENGINE = ReplicatedReplacingMergeTree"}, } { - tempDir, err := processSchemaDir(schemaDir, "otel", c.replicated, map[string]string{}) + tempDir, err := processSchemaDir(schemaDir, "otel", c.replicated, c.replicatedTimeSeries, map[string]string{}) if err != nil { t.Fatal(err) } @@ -892,15 +895,29 @@ func TestProcessSchemaDir_ReplicatedMacros(t *testing.T) { t.Fatal(err) } if string(got) != c.want { - t.Errorf("replicated=%v: got %q, want %q", c.replicated, string(got), c.want) + t.Errorf("replicated=%v replicatedTimeSeries=%v: got %q, want %q", c.replicated, c.replicatedTimeSeries, string(got), c.want) + } + } +} + +func TestSupportsReplicatedTimeSeries(t *testing.T) { + for _, c := range []struct { + major, minor int + want bool + }{ + {25, 12, false}, {26, 4, false}, {26, 8, false}, {26, 10, true}, {26, 11, true}, {27, 1, true}, + } { + if got := supportsReplicatedTimeSeries(c.major, c.minor); got != c.want { + t.Errorf("supportsReplicatedTimeSeries(%d, %d) = %v; want %v", c.major, c.minor, got, c.want) } } } // Guards the real seed files: every MergeTree-family engine must use the // ${ENGINE_PREFIX} macro and take no arguments (a Replicated database rejects -// explicit Keeper path/replica arguments), and the database must be created -// with the Replicated engine only when enabled. +// explicit Keeper path/replica arguments), every TimeSeries table must get +// Replicated inner engines exactly when replicated, and the database must be +// created with the Replicated engine only when enabled. func TestSeedSchemaEngines(t *testing.T) { seedDir := filepath.Join("..", "..", "..", "..", "docker", "otel-collector", "schema", "seed") ttlExprs := map[string]string{} @@ -908,8 +925,9 @@ func TestSeedSchemaEngines(t *testing.T) { ttlExprs[m] = "toIntervalDay(30)" } engineRe := regexp.MustCompile(`(?im)^\s*ENGINE\s*=\s*(.*)$`) + innerEngineRe := regexp.MustCompile(`(?im)^\s*(?:DATA|TAGS|METRICS)\s+ENGINE\s*=\s*(\w+)`) for _, replicated := range []bool{false, true} { - tempDir, err := processSchemaDir(seedDir, "otel", replicated, ttlExprs) + tempDir, err := processSchemaDir(seedDir, "otel", replicated, replicated, ttlExprs) if err != nil { t.Fatal(err) } @@ -918,7 +936,7 @@ func TestSeedSchemaEngines(t *testing.T) { if err != nil { t.Fatal(err) } - mergeTrees := 0 + mergeTrees, timeSeries := 0, 0 for _, f := range files { content, err := os.ReadFile(filepath.Join(tempDir, f)) if err != nil { @@ -927,9 +945,11 @@ func TestSeedSchemaEngines(t *testing.T) { if strings.Contains(string(content), "${") { t.Errorf("replicated=%v: %s has an unreplaced macro", replicated, f) } + fileTimeSeries := 0 for _, m := range engineRe.FindAllStringSubmatch(string(content), -1) { engine := strings.TrimSpace(m[1]) if engine == "TimeSeries" { + fileTimeSeries++ continue } mergeTrees++ @@ -940,10 +960,28 @@ func TestSeedSchemaEngines(t *testing.T) { t.Errorf("replicated=%v: %s has engine %q", replicated, f, engine) } } + // Without replication, inner engines stay unnamed so each server keeps its own defaults. + inner := innerEngineRe.FindAllStringSubmatch(string(content), -1) + wantInner := 0 + if replicated { + wantInner = 3 * fileTimeSeries + } + if len(inner) != wantInner { + t.Errorf("replicated=%v: %s has %d TimeSeries inner engines; want %d", replicated, f, len(inner), wantInner) + } + for _, m := range inner { + if !strings.HasPrefix(m[1], "Replicated") || !strings.HasSuffix(m[1], "MergeTree") { + t.Errorf("%s: TimeSeries inner engine %q must be Replicated*MergeTree", f, m[1]) + } + } + timeSeries += fileTimeSeries } if mergeTrees == 0 { t.Errorf("replicated=%v: no MergeTree engines found in %s", replicated, seedDir) } + if timeSeries == 0 { + t.Errorf("replicated=%v: no TimeSeries engines found in %s", replicated, seedDir) + } createDB, err := os.ReadFile(filepath.Join(tempDir, "00001_create_database.sql")) if err != nil { @@ -1496,7 +1534,7 @@ SETTINGS ttl_only_drop_parts = 1;` t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "mydb", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "mydb", false, false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) } From ca2e4a3113bf50287250a27b36ffe4b9b017d913 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Fri, 20 Nov 2026 18:17:45 -0500 Subject: [PATCH 4/4] fix: point the Replicated warning at the fallback setting when the fallback wins --- packages/otel-collector/cmd/migrate/main.go | 10 +++-- .../otel-collector/cmd/migrate/main_test.go | 37 ++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 1108ddaaa2..dff3068e20 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -628,13 +628,17 @@ func processSchemaDir(schemaDir, database string, replicated, replicatedTimeSeri // planReplicated decides whether to seed Replicated engines. Argument-less // Replicated* table engines only work inside a Replicated database and an // existing database cannot change engine, so only a missing or already -// Replicated database qualifies. Returns a warning when the flag cannot apply. -func planReplicated(enabled, exists bool, database, engine string) (bool, string) { +// Replicated database qualifies. Returns a warning when the flag cannot apply; +// database differs from target when the fallback was chosen instead. +func planReplicated(enabled, exists bool, target, database, engine string) (bool, string) { switch { case !enabled: return false, "" case !exists, engine == "Replicated": return true, "" + case database != target: + return false, fmt.Sprintf("fallback database %s has engine %s, so tables use non-replicated engines; "+ + "clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to seed a Replicated database %s instead", database, engine, target) default: return false, fmt.Sprintf("database %s already exists with engine %s, so tables use non-replicated engines; "+ "set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE to a new database name to seed a Replicated database", database, engine) @@ -1012,7 +1016,7 @@ func main() { log.Fatalf("Failed to check database engine: %v", err) } } - replicated, warning := planReplicated(cfg.Replicated, dbExists, database, dbEngine) + replicated, warning := planReplicated(cfg.Replicated, dbExists, cfg.Database, database, dbEngine) if warning != "" { log.Printf("WARNING: %s", warning) } diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 56ae17bac5..05708bd794 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -1001,27 +1001,38 @@ func TestPlanReplicated(t *testing.T) { cases := []struct { name string enabled, exists bool + database string engine string - want, warns bool + want bool + wantWarning []string // substrings; nil means no warning }{ - {"flag off", false, true, "Atomic", false, false}, - {"flag off, database missing", false, false, "", false, false}, - {"flag off, database Replicated", false, true, "Replicated", false, false}, - {"database missing", true, false, "", true, false}, - {"database Replicated", true, true, "Replicated", true, false}, - {"database Atomic", true, true, "Atomic", false, true}, - {"database Shared", true, true, "Shared", false, true}, + {"flag off", false, true, "otel", "Atomic", false, nil}, + {"flag off, database missing", false, false, "otel", "", false, nil}, + {"flag off, database Replicated", false, true, "otel", "Replicated", false, nil}, + {"database missing", true, false, "otel", "", true, nil}, + {"database Replicated", true, true, "otel", "Replicated", true, nil}, + {"database Atomic", true, true, "otel", "Atomic", false, + []string{"database otel", "engine Atomic", "set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE to a new database name"}}, + {"database Shared", true, true, "otel", "Shared", false, []string{"database otel", "engine Shared"}}, + {"fallback Replicated", true, true, "default", "Replicated", true, nil}, + {"fallback Atomic", true, true, "default", "Atomic", false, + []string{"fallback database default", "engine Atomic", "clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", "Replicated database otel"}}, } for _, c := range cases { - got, warning := planReplicated(c.enabled, c.exists, "otel", c.engine) + got, warning := planReplicated(c.enabled, c.exists, "otel", c.database, c.engine) if got != c.want { t.Errorf("%s: replicated = %v; want %v", c.name, got, c.want) } - if (warning != "") != c.warns { - t.Errorf("%s: warning = %q; want warning=%v", c.name, warning, c.warns) + if c.wantWarning == nil && warning != "" { + t.Errorf("%s: unexpected warning %q", c.name, warning) } - if c.warns && (!strings.Contains(warning, c.engine) || !strings.Contains(warning, "otel")) { - t.Errorf("%s: warning should name the database and engine: %q", c.name, warning) + if c.wantWarning != nil && warning == "" { + t.Errorf("%s: want a warning", c.name) + } + for _, sub := range c.wantWarning { + if !strings.Contains(warning, sub) { + t.Errorf("%s: warning %q should contain %q", c.name, warning, sub) + } } } }