diff --git a/.changeset/mv-quantile-sketch-accuracy.md b/.changeset/mv-quantile-sketch-accuracy.md new file mode 100644 index 0000000000..25bd74a2df --- /dev/null +++ b/.changeset/mv-quantile-sketch-accuracy.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +fix: read percentiles correctly from materialized views that store DDSketch (`quantileDD`) or GK (`quantileGK`) states. The merge now passes the sketch's accuracy ahead of the level, so charts and alerts no longer return a different percentile without an error, and GK views are no longer skipped. diff --git a/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts b/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts new file mode 100644 index 0000000000..cce58fbaf3 --- /dev/null +++ b/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts @@ -0,0 +1,216 @@ +import type { ClickHouseClient } from '@clickhouse/client'; +import { createClient } from '@clickhouse/client'; + +import { parameterizedQueryToSql } from '@/clickhouse'; +import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; +import { tryOptimizeConfigWithMaterializedViewWithExplanations } from '@/core/materializedViews'; +import { Metadata, MetadataCache } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { + BuilderChartConfigWithOptDateRange, + DisplayType, + SourceKind, + TTraceSource, +} from '@/types'; + +// Percentiles read from a materialized view have to match the raw table +// whichever quantile sketch the view stores. Sketches built with an accuracy +// (DDSketch, GK) only merge correctly when the merge is given that accuracy +// ahead of the level. +describe('materialized view quantile sketches (integration)', () => { + let client: ClickHouseClient; + let hdxClient: HdxClickhouseClient; + let metadata: Metadata; + + const DB = 'default'; + const SPANS_TABLE = 'test_mv_sketch_spans'; + const SKETCHES = [ + { name: 'quantileDD', params: '0.01, 0.5', tolerance: 0.01 }, + { name: 'quantileGK', params: '1000, 0.5', tolerance: 0.01 }, + { name: 'quantileTDigest', params: '0.5', tolerance: 0.02 }, + { name: 'quantileBFloat16', params: '0.5', tolerance: 0.01 }, + ]; + const rollupTable = (sketch: string) => `test_mv_sketch_rollup_${sketch}`; + + const sourceFor = (sketch: string) => + ({ + id: 'test-traces', + kind: SourceKind.Trace, + name: 'Traces', + connection: 'test-connection', + from: { databaseName: DB, tableName: SPANS_TABLE }, + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + materializedViews: [ + { + databaseName: DB, + tableName: rollupTable(sketch), + dimensionColumns: 'ServiceName', + minGranularity: '1 minute', + timestampColumn: 'Timestamp', + aggregatedColumns: [ + { aggFn: 'count', mvColumn: 'count', sourceColumn: '' }, + { + aggFn: 'quantile', + mvColumn: 'quantile__Duration', + sourceColumn: 'Duration', + }, + ], + }, + ], + }) as TTraceSource; + + // Five-minute buckets, so each bucket merges five one-minute sketch states. + const percentileConfig = ( + level: number, + ): BuilderChartConfigWithOptDateRange => ({ + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DB, tableName: SPANS_TABLE }, + select: [ + { + alias: 'percentile', + aggFn: 'quantile', + level, + valueExpression: 'Duration', + aggCondition: '', + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-06-01T00:00:00Z'), + new Date('2025-06-01T00:10:00Z'), + ], + granularity: '5 minute', + }); + + async function queryRows(sql: string) { + const result = await client.query({ query: sql, format: 'JSONEachRow' }); + return (await result.json()) as Record[]; + } + + beforeAll(async () => { + const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123'; + const username = process.env.CLICKHOUSE_USER || 'default'; + const password = process.env.CLICKHOUSE_PASSWORD || ''; + + client = createClient({ url: host, username, password }); + hdxClient = new HdxClickhouseClient({ host, username, password }); + + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${DB}.${SPANS_TABLE} ( + Timestamp DateTime64(9), + ServiceName LowCardinality(String), + Duration UInt64 + ) + ENGINE = MergeTree() + ORDER BY (Timestamp) + `, + }); + // 100,000 spans over ten minutes. Durations from 1 microsecond to + // 100 milliseconds are permuted across the spans, so every minute holds + // a spread of the whole range. + await client.command({ + query: ` + INSERT INTO ${DB}.${SPANS_TABLE} + SELECT + toDateTime64('2025-06-01 00:00:00', 9, 'UTC') + toIntervalMillisecond(number * 6), + 'api', + ((number * 7919) % 100000 + 1) * 1000 + FROM numbers(100000) + `, + }); + + for (const { name, params } of SKETCHES) { + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${DB}.${rollupTable(name)} ( + Timestamp DateTime, + ServiceName LowCardinality(String), + count SimpleAggregateFunction(sum, UInt64), + quantile__Duration AggregateFunction(${name}(${params}), UInt64) + ) + ENGINE = AggregatingMergeTree() + ORDER BY (Timestamp, ServiceName) + `, + }); + await client.command({ + query: ` + INSERT INTO ${DB}.${rollupTable(name)} + SELECT + toStartOfMinute(Timestamp) AS Timestamp, + ServiceName, + count() AS count, + ${name}State(${params})(Duration) AS quantile__Duration + FROM ${DB}.${SPANS_TABLE} + GROUP BY Timestamp, ServiceName + `, + }); + } + }); + + beforeEach(() => { + metadata = new Metadata(hdxClient, new MetadataCache()); + }); + + afterAll(async () => { + for (const { name } of SKETCHES) { + await client.command({ + query: `DROP TABLE IF EXISTS ${DB}.${rollupTable(name)}`, + }); + } + await client.command({ + query: `DROP TABLE IF EXISTS ${DB}.${SPANS_TABLE}`, + }); + await hdxClient.close(); + await client.close(); + }); + + it.each(SKETCHES)( + 'reads p95 and p99 from a $name rollup within its error of the raw table', + async ({ name, tolerance }) => { + for (const level of [0.95, 0.99]) { + const { optimizedConfig } = + await tryOptimizeConfigWithMaterializedViewWithExplanations( + percentileConfig(level), + metadata, + hdxClient, + undefined, + sourceFor(name), + ); + expect(optimizedConfig?.from.tableName).toBe(rollupTable(name)); + if (optimizedConfig == null) { + return; + } + + const rollupRows = await queryRows( + parameterizedQueryToSql( + await renderChartConfig(optimizedConfig, metadata, undefined), + ), + ); + const exactRows = await queryRows(` + SELECT + toStartOfInterval(toDateTime(Timestamp), INTERVAL 5 MINUTE) AS __hdx_time_bucket, + quantileExact(${level})(Duration) AS percentile + FROM ${DB}.${SPANS_TABLE} + GROUP BY __hdx_time_bucket + `); + const exactByBucket = new Map( + exactRows.map(row => [row.__hdx_time_bucket, Number(row.percentile)]), + ); + + expect(rollupRows).toHaveLength(2); + for (const row of rollupRows) { + const exact = exactByBucket.get(row.__hdx_time_bucket); + expect(exact).toBeDefined(); + const relativeError = + Math.abs(Number(row.percentile) - (exact ?? 0)) / (exact ?? 1); + expect(relativeError).toBeLessThanOrEqual(tolerance); + } + } + }, + ); +}); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index e048142170..6e098084f5 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3396,6 +3396,58 @@ describe('renderChartConfig', () => { expect(actual).toMatchSnapshot(); }); + it('should merge a quantile sketch with its accuracy ahead of the level', async () => { + // Built the way the materialized view optimizer builds it: a quantile + // select item with its aggFn swapped for the sketch's merge function. + const sketchMerge = ( + level: number, + condition?: { aggCondition: string; aggConditionLanguage: 'lucene' }, + ) => ({ + ...{ + aggFn: 'quantile' as const, + level, + valueExpression: 'quantile__Duration', + aggCondition: '', + ...condition, + }, + aggFn: 'quantileDDMerge', + sketchParams: [0.01], + }); + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [ + sketchMerge(0.95), + sketchMerge(0.99, { + aggCondition: 'severity:"ERROR"', + aggConditionLanguage: 'lucene', + }), + ], + where: '', + whereLanguage: 'sql', + groupBy: 'severity', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain( + 'quantileDDMerge(0.01, 0.95)(quantile__Duration)', + ); + expect(actual).toContain( + "quantileDDMergeIf(0.01, 0.99)(quantile__Duration, ((severity = 'ERROR')) AND", + ); + }); + it('should generate SQL for an histogram merge function', async () => { const config: ChartConfigWithOptDateRange = { displayType: DisplayType.Table, diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index f25ad35367..f44c8bffb5 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -399,6 +399,111 @@ describe('materializedViews', () => { expect(result.errors).toBeUndefined(); }); + it.each([ + { + type: 'AggregateFunction(quantileDD(0.01, 0.5), UInt64)', + aggFn: 'quantileDDMerge', + sketchParams: [0.01], + }, + { + type: 'AggregateFunction(quantilesDD(0.001, 0.9, 0.99), UInt64)', + aggFn: 'quantileDDMerge', + sketchParams: [0.001], + }, + { + type: 'AggregateFunction(quantileGK(100, 0.5), UInt64)', + aggFn: 'quantileGKMerge', + sketchParams: [100], + }, + ])( + 'should carry the accuracy of a $type select into the merge', + async ({ type, aggFn, sketchParams }) => { + const sketchMetadata: Metadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + if (column === 'quantile__Duration') { + return { type } as unknown as ColumnMeta; + } + return undefined; + }), + } as unknown as Metadata; + + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: 'Duration', + aggFn: 'quantile', + level: 0.95, + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + sketchMetadata, + ); + + expect(result.optimizedConfig?.select).toEqual([ + { + valueExpression: 'quantile__Duration', + aggFn, + level: 0.95, + sketchParams, + }, + ]); + expect(result.errors).toBeUndefined(); + }, + ); + + it('should not carry sketch parameters for sketches merged by level alone', async () => { + const bfloat16Metadata: Metadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + if (column === 'quantile__Duration') { + return { + type: 'AggregateFunction(quantileBFloat16(0.5), UInt64)', + } as unknown as ColumnMeta; + } + return undefined; + }), + } as unknown as Metadata; + + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: 'Duration', + aggFn: 'quantile', + level: 0.99, + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + bfloat16Metadata, + ); + + expect(result.optimizedConfig?.select).toEqual([ + { + valueExpression: 'quantile__Duration', + aggFn: 'quantileBFloat16Merge', + level: 0.99, + }, + ]); + }); + it('should convert a histogram AggregateFunction select', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index e7ba7d132d..1b1490f69c 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -221,12 +221,19 @@ async function isSimpleAggregateFunction( } } +// Quantile sketches whose state is built with an accuracy ahead of the level, +// e.g. quantileDD(relative_accuracy, level) and quantileGK(accuracy, level). +// Their -Merge has to be given the same accuracy: with the level alone, +// ClickHouse reads the level as the accuracy and returns a different quantile +// without raising an error. +const ACCURACY_PARAMETERIZED_QUANTILES = new Set(['quantileDD', 'quantileGK']); + // Variants of quantile (ex. quantileExact, quantileDD, etc.) async function getQuantileAggregateFunction( tableConnection: TableConnection, column: string, metadata: Metadata, -) { +): Promise<{ name: string; sketchParams?: number[] } | undefined> { try { const columnMeta = await metadata.getColumn({ ...tableConnection, @@ -238,16 +245,31 @@ async function getQuantileAggregateFunction( return undefined; } - // Use regex to extract the quantile function name inside AggregateFunction(...) + // Use regex to extract the quantile function name and its parameters inside AggregateFunction(...) // For example, AggregateFunction(quantile(0.95), Int64) --> quantile // AggregateFunction(quantileTDigest(0.95), Int64) --> quantileTDigest - // AggregateFunction(quantileDD(0.001, 0.95), Int64) --> quantileDD + // AggregateFunction(quantileDD(0.001, 0.95), Int64) --> quantileDD, accuracy 0.001 // The plural `quantiles*` variants return arrays, but a select item carries a // single `level`, so we normalize to the singular form to pull a scalar value. // AggregateFunction(quantiles(0.9, 0.95), UInt64) --> quantile // AggregateFunction(quantilesTDigest(0.9, 0.95), UInt64) --> quantileTDigest - const match = type.match(/^AggregateFunction\(\s*([^(, ]+)\s*\(/); - return match?.[1]?.replace(/^quantiles/, 'quantile'); + // AggregateFunction(quantilesDD(0.01, 0.9, 0.95), UInt64) --> quantileDD, accuracy 0.01 + const match = type.match(/^AggregateFunction\(\s*([^(, ]+)\s*\(([^)]*)\)/); + const name = match?.[1]?.replace(/^quantiles/, 'quantile'); + if (!name) { + return undefined; + } + if (!ACCURACY_PARAMETERIZED_QUANTILES.has(name)) { + return { name }; + } + + // The accuracy is the first parameter in both the singular and plural forms. + // Without it the merge can't match the state, so leave the select item + // unconverted and let the EXPLAIN check reject this view. + const accuracy = Number(match?.[2]?.split(',')[0]?.trim()); + return Number.isFinite(accuracy) && accuracy > 0 + ? { name, sketchParams: [accuracy] } + : undefined; } catch { return undefined; } @@ -439,6 +461,7 @@ async function convertSelectToMaterializedViewSelect( ); } + let sketchParams: number[] | undefined; if (isQuantileSelectItem(selectItem)) { const quantileAggregateFunction = await getQuantileAggregateFunction( mvTableConnection, @@ -446,7 +469,8 @@ async function convertSelectToMaterializedViewSelect( metadata, ); if (quantileAggregateFunction) { - aggFn = quantileAggregateFunction; + aggFn = quantileAggregateFunction.name; + sketchParams = quantileAggregateFunction.sketchParams; } } @@ -461,6 +485,7 @@ async function convertSelectToMaterializedViewSelect( ...selectItem, valueExpression: aggregatedColumnConfig.mvColumn, aggFn: aggMergeFn, + ...(sketchParams != null ? { sketchParams } : {}), }; } diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 95c696b4d9..30691de7f5 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -590,6 +590,7 @@ const aggFnExpr = ({ fn, expr, level, + sketchParams, where, sampleWeightExpression, keepValueType = false, @@ -597,6 +598,8 @@ const aggFnExpr = ({ fn: AggregateFunction | AggregateFunctionWithCombinators; expr?: string; level?: number; + /** Parameters of a quantile sketch state that precede the level, for -Merge. */ + sketchParams?: number[]; where?: string; sampleWeightExpression?: string; /** Aggregate `expr` as-is instead of casting it to Float64. */ @@ -620,9 +623,17 @@ const aggFnExpr = ({ const shouldParameterizeWithLevel = level && (fn.startsWith('quantile') || fn.startsWith('histogram')); + // A sketch built with an accuracy (quantileDD, quantileGK) merges with + // that accuracy ahead of the level, e.g. quantileDDMerge(0.01, 0.95). + const mergeParams = [ + ...(fn.startsWith('quantile') ? (sketchParams ?? []) : []), + level, + ]; const renderedFnArgsWithQuantileLevel = shouldParameterizeWithLevel ? chSql`(${{ - UNSAFE_RAW_SQL: Number.isFinite(level) ? `${level}` : '0', + UNSAFE_RAW_SQL: mergeParams + .map(param => (Number.isFinite(param) ? `${param}` : '0')) + .join(', '), }})` : []; @@ -863,6 +874,7 @@ async function renderSelectList( expr: select.valueExpression, // @ts-expect-error (TS doesn't know that we've already checked for quantile) level: select.level, + sketchParams: select.sketchParams, where: whereClause.sql, sampleWeightExpression: chartConfig.sampleWeightExpression, }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 73e0236caf..7b681c4d36 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -481,6 +481,11 @@ export const DerivedColumnSchema = z.intersection( // `ColorConditionSchema` so no schema change was needed (the union's // string-match operators were always intended for this slice). colorRules: z.array(ColorConditionSchema).max(10).optional(), + // Parameters a materialized view's quantile sketch was built with, other + // than the level: the accuracy of quantileDD(0.01, 0.5) or + // quantileGK(100, 0.5). The materialized view optimizer sets it so the + // -Merge call matches the stored state; it is not user-configurable. + sketchParams: z.array(z.number()).optional(), }), ); export const SelectListSchema = z.array(DerivedColumnSchema).or(z.string());