From 9eab5a85b1e490c6ecbd9de6672473982ee1a93c Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:56:13 -0500 Subject: [PATCH 01/27] fix: drop the row cap from text index value lookups --- .changeset/text-index-values-no-row-cap.md | 11 ++++++++ .../src/__tests__/metadata.test.ts | 26 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 16 +++++------- 3 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 .changeset/text-index-values-no-row-cap.md diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md new file mode 100644 index 0000000000..103ed81b01 --- /dev/null +++ b/.changeset/text-index-values-no-row-cap.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': patch +--- + +fix: show values for every key in text index value lookups + +"All values" lookups that read a text index (Map key=value indexes and text +indexes on plain columns) stopped at `max_rows_to_read` with +`read_overflow_mode = 'break'`. The index is read in sorted token order, so once +a high-cardinality key used up the budget, every key that sorts after it came +back with no values and no error. These lookups no longer apply a row cap. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 8ea6193746..74cc3a91eb 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1403,6 +1403,32 @@ describe('Metadata', () => { expect(sql).toContain('GROUP BY column, key'); }); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads the %s text index without a row cap, even when one is configured', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + jest + .spyOn(metadata, 'getClickHouseSettings') + .mockReturnValue({ max_rows_to_read: '1000' }); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + }); + + const calls = (mockClickhouseClient.query as jest.Mock).mock.calls; + const textIndexCall = calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].clickhouse_settings).toMatchObject({ + max_rows_to_read: '0', + }); + }, + ); + it.each([ ['map KV', "LogAttributes['requestId']"], ['native', 'TraceId'], diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 81a93eedad..51f1825de3 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1819,12 +1819,11 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // The dictionary is read in sorted token order, so a row cap + // with read_overflow_mode=break silently leaves every key that + // sorts after the cutoff with no values. + max_rows_to_read: '0', }, abort_signal: signal, }) @@ -1902,12 +1901,9 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // See getMapTextIndexKeyValues. + max_rows_to_read: '0', }, abort_signal: signal, }) From 0a6b420d4afaf3cf2d0dfb69109f22ac68aeaab6 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:02:35 -0500 Subject: [PATCH 02/27] feat: show the most frequent text index values with topKWeighted --- .changeset/topk-text-index-values.md | 11 ++++++++ .../src/__tests__/metadata.int.test.ts | 27 +++++++++++++++++++ .../src/__tests__/metadata.test.ts | 23 +++++++--------- packages/common-utils/src/core/metadata.ts | 6 +++-- 4 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 .changeset/topk-text-index-values.md diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md new file mode 100644 index 0000000000..332e05a55d --- /dev/null +++ b/.changeset/topk-text-index-values.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': patch +--- + +feat: show the most frequent filter values read from text indexes + +"All values" lookups that read a text index (Map key=value indexes and text +indexes on plain columns) used `groupUniqArray(limit)`, which keeps the +alphabetically first tokens it reads. They now use `topKWeighted(limit)` +weighted by each token's row count, so a capped list holds the most common +values, most common first. This costs more CPU per token than before. diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 0e9133bc87..25ff349cf5 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -907,6 +907,33 @@ describe('Metadata Integration Tests', () => { ); }); + it('keeps the most frequent text index values when maxValuesPerKey truncates the list', async () => { + if (!textIndexSupported) { + console.warn( + 'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()', + ); + return; + } + // The index dictionary is sorted, so 'a-rare' is read first. + const freqKey = `freq.${tag}`; + const timestamp = testTime.toISOString().replace('T', ' ').slice(0, 23); + await client.command({ + query: `INSERT INTO default.otel_logs (Timestamp, ResourceAttributes) + SELECT '${timestamp}', map('${freqKey}', if(number = 0, 'a-rare', 'z-common')) + FROM numbers(4)`, + }); + + const result = await metadata.getAllKeyValues({ + ...commonArgs, + keyExpressions: [`ResourceAttributes['${freqKey}']`], + maxValuesPerKey: 1, + }); + + expect(result).toEqual([ + { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, + ]); + }); + it('routes TraceId through getTextIndexKeyValues', async () => { if (!textIndexSupported) { console.warn( diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 74cc3a91eb..e30fa0c101 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1380,7 +1380,7 @@ describe('Metadata', () => { const sql = calls[calls.length - 1][0].query as string; expect(sql).toContain('mergeTreeTextIndex('); - expect(sql).toContain('groupUniqArray('); + expect(sql).toContain('topKWeighted('); expect(sql).toContain('GROUP BY key'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1831,9 +1831,8 @@ describe('Metadata', () => { // via text index or raw scan. describe('maxValuesPerKey threading', () => { // Pick a value that (a) isn't ambient in the test setup (like 20, - // dates, or granularity numbers) and (b) is easy to spot in a param - // dump when a test fails. Int32 chSql params render as numbers, so - // assertions compare against the numeric literal. + // dates, or granularity numbers) and (b) is easy to spot in the SQL or a + // param dump when a test fails. const MAX_VALUES = 7777; it('threads maxValuesPerKey into the map text-index query', async () => { @@ -1853,11 +1852,9 @@ describe('Metadata', () => { c[0].query.includes('startsWith(token,'), ); expect(mapTextIndexCall).toBeDefined(); - const params = mapTextIndexCall![0].query_params as Record< - string, - unknown - >; - expect(Object.values(params)).toContain(MAX_VALUES); + expect(mapTextIndexCall![0].query).toContain( + `topKWeighted(${MAX_VALUES})(`, + ); }); it('threads maxValuesPerKey into the native text-index query', async () => { @@ -1878,11 +1875,9 @@ describe('Metadata', () => { !c[0].query.includes('startsWith(token,'), ); expect(nativeTextIndexCall).toBeDefined(); - const params = nativeTextIndexCall![0].query_params as Record< - string, - unknown - >; - expect(Object.values(params)).toContain(MAX_VALUES); + expect(nativeTextIndexCall![0].query).toContain( + `topKWeighted(${MAX_VALUES})(token, cardinality)`, + ); }); it('threads maxValuesPerKey into the raw-table getKeyValues fallback', async () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 51f1825de3..7cb7218395 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1798,11 +1798,13 @@ export class Metadata { partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; + // `cardinality` is the token's row count in the part, so this keeps + // each key's most frequent values rather than the first ones read. const sql = chSql` SELECT * FROM ( SELECT ${{ String: columnName }} as column, substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, - groupUniqArray(${{ Int32: info.limit }})(${valueSql}) AS value + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND (${orChain}) @@ -1885,7 +1887,7 @@ export class Metadata { const sql = chSql` SELECT * FROM ( SELECT ${{ String: columnName }} AS key, - groupUniqArray(${{ Int32: info.limit }})(token) AS value + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND token != '' From f70247250fac699058b772b8a5f23e4597323622 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:31:49 -0500 Subject: [PATCH 03/27] perf: read text index values per key with a token cap --- .changeset/topk-text-index-values.md | 16 +++-- .../src/__tests__/metadata.int.test.ts | 16 +++-- .../src/__tests__/metadata.test.ts | 30 ++++++++- packages/common-utils/src/core/metadata.ts | 66 ++++++++++--------- 4 files changed, 85 insertions(+), 43 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 332e05a55d..a25ccb0f78 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,10 +2,16 @@ '@hyperdx/common-utils': patch --- -feat: show the most frequent filter values read from text indexes +perf: read each key's text index values separately, and show the most +frequent ones "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) used `groupUniqArray(limit)`, which keeps the -alphabetically first tokens it reads. They now use `topKWeighted(limit)` -weighted by each token's row count, so a capped list holds the most common -values, most common first. This costs more CPU per token than before. +indexes on plain columns) read every token of every requested key, capped at +`max_rows_to_read` for the whole query. One key with a value per row (request +IDs, UUIDs) slowed the whole query down, and because the index is read in +sorted order, the cap left keys that sort after it with no values. Each key now +has its own query branch that reads at most 100,000 tokens, and there is no +whole-query row cap. Values come from `topKWeighted(limit)` weighted by each +token's row count, so a capped list holds the most common values, most common +first. For keys with more distinct values than the token cap, they are the most +common among the alphabetically first ones. diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 25ff349cf5..82778f25e5 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -925,13 +925,21 @@ describe('Metadata Integration Tests', () => { const result = await metadata.getAllKeyValues({ ...commonArgs, - keyExpressions: [`ResourceAttributes['${freqKey}']`], + keyExpressions: [ + `ResourceAttributes['${freqKey}']`, + `ResourceAttributes['${mapKey}']`, + ], maxValuesPerKey: 1, }); - expect(result).toEqual([ - { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, - ]); + // UNION ALL branches can arrive in any order. + expect(result).toHaveLength(2); + expect(result).toEqual( + expect.arrayContaining([ + { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, + { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] }, + ]), + ); }); it('routes TraceId through getTextIndexKeyValues', async () => { diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e30fa0c101..c66904a2fe 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1381,7 +1381,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - expect(sql).toContain('GROUP BY key'); + expect(sql).toContain('LIMIT 100000'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1400,7 +1400,33 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('startsWith(token,'); expect(sql).toContain('substring(token, position(token,'); - expect(sql).toContain('GROUP BY column, key'); + expect(sql).toContain("startsWith(token, 'requestId=')"); + }); + + it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['region']", + ], + }); + + const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find( + ([opts]) => (opts.query as string).includes('startsWith(token,'), + )![0]; + const branches = (call.query as string).split('UNION ALL'); + expect(branches).toHaveLength(2); + expect(branches[0]).toContain("startsWith(token, 'requestId=')"); + expect(branches[0]).toContain("'requestId' AS key"); + expect(branches[1]).toContain("startsWith(token, 'region=')"); + for (const branch of branches) { + expect(branch).toContain('LIMIT 100000'); + } + expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); + expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); }); it.each([ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 7cb7218395..783434c537 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -78,6 +78,11 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; +// Tokens a text index values lookup reads per key. The dictionary is sorted, +// so a key with more distinct values than this gets the most frequent among +// its alphabetically first ones. Keep it above "Load more"'s 10,000 values. +const TEXT_INDEX_TOKENS_PER_KEY = 100_000; + /** * The index read could not be scoped to the requested date range, so it was * refused in favour of a path that can. Not a failure — an expected property @@ -1776,19 +1781,6 @@ export class Metadata { try { const sqlBranches: Array = []; for (const [columnName, info] of queryOptions.entries()) { - const orChain = concatChSql( - ' OR ', - // Inline keys as SQL-escaped literals, not bind params: ~100 - // per-key params exceed the web client's URL param budget and - // silently switch the request to a multipart body that proxies - // may reject. Keys are ingest-controlled, hence SqlString.escape. - info.keys.map( - k => - chSql`startsWith(token, ${{ - UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`), - }})`, - ), - ); const partsFilter = await this.partsOverlapFilter({ databaseName, tableName, @@ -1798,20 +1790,29 @@ export class Metadata { partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; - // `cardinality` is the token's row count in the part, so this keeps - // each key's most frequent values rather than the first ones read. - const sql = chSql` - SELECT * FROM ( - SELECT ${{ String: columnName }} as column, - substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value + // One branch per key, each with its own LIMIT: the read for a key + // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a + // value per row can't slow down or starve the others. `cardinality` + // is the token's row count in the part, so topKWeighted keeps the + // most frequent values among the tokens read. + for (const k of info.keys) { + // Inline keys as SQL-escaped literals, not bind params: ~100 + // per-key params exceed the web client's URL param budget and + // silently switch the request to a multipart body that proxies + // may reject. Keys are ingest-controlled, hence SqlString.escape. + sqlBranches.push(chSql` + SELECT ${{ String: columnName }} AS column, + ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value + FROM ( + SELECT ${valueSql} AS v, cardinality FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} - AND (${orChain}) - AND ${valueSql} != '' - GROUP BY column, key - )`; - sqlBranches.push(sql); + AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}) + AND v != '' + LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} + )`); + } } const sql = concatChSql(' UNION ALL ', sqlBranches); @@ -1822,9 +1823,8 @@ export class Metadata { connectionId, clickhouse_settings: { ...this.getClickHouseSettings(), - // The dictionary is read in sorted token order, so a row cap - // with read_overflow_mode=break silently leaves every key that - // sorts after the cutoff with no values. + // The per-key LIMIT bounds the read. A row cap would stop the + // sorted dictionary read partway and leave later keys empty. max_rows_to_read: '0', }, abort_signal: signal, @@ -1884,14 +1884,16 @@ export class Metadata { maxParts: maxParts ?? undefined, partSelection, }); + // See getMapTextIndexKeyValues for the per-column LIMIT. const sql = chSql` - SELECT * FROM ( - SELECT ${{ String: columnName }} AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value + SELECT ${{ String: columnName }} AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value + FROM ( + SELECT token, cardinality FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND token != '' - GROUP BY key + LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} )`; sqlBranches.push(sql); } From cd97c8070be06564a44ef8562e21e2ed7bfb580c Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:40:01 -0500 Subject: [PATCH 04/27] perf: run text index values in batches of 10 keys --- .../src/__tests__/metadata.test.ts | 18 ++++++++ packages/common-utils/src/core/metadata.ts | 41 +++++++++++-------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index c66904a2fe..e42e62de12 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1537,6 +1537,24 @@ describe('Metadata', () => { }, ); + it('splits Map text index values into queries of at most 10 keys', async () => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: Array.from( + { length: 25 }, + (_, i) => `LogAttributes['key${i}']`, + ), + }); + + const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls + .map(([opts]) => opts.query as string) + .filter(q => q.includes('startsWith(token,')) + .map(q => q.split('UNION ALL').length); + expect(branchCounts).toEqual([10, 10, 5]); + }); + it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 783434c537..90276fe1dd 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -82,6 +82,8 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; // so a key with more distinct values than this gets the most frequent among // its alphabetically first ones. Keep it above "Load more"'s 10,000 values. const TEXT_INDEX_TOKENS_PER_KEY = 100_000; +// Keys per text index values query; see dispatchKeyValueQueries. +const TEXT_INDEX_KEYS_PER_QUERY = 10; /** * The index read could not be scoped to the requested date range, so it was @@ -3060,28 +3062,35 @@ export class Metadata { // fire all the kv fetch queries const promises: Array> = []; - if (mapTextIndexQueryOptions.size > 0) { - promises.push( - this.getMapTextIndexKeyValues({ - databaseName, - tableName, - connectionId, - queryOptions: mapTextIndexQueryOptions, - maxParts: mapKeyIndexMaxParts, - partSelection: mapKeyIndexPartSelection, - dateRange, - timestampValueExpression, - signal, - }), - ); + // Text index lookups run in small parallel batches, so a slow, + // high-cardinality key holds up only its own batch's values. + for (const [column, options] of mapTextIndexQueryOptions) { + for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) { + promises.push( + this.getMapTextIndexKeyValues({ + databaseName, + tableName, + connectionId, + queryOptions: new Map([[column, { ...options, keys }]]), + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, + dateRange, + timestampValueExpression, + signal, + }), + ); + } } - if (nativeTextIndexQueryOptions.size > 0) { + for (const entries of chunk( + [...nativeTextIndexQueryOptions], + TEXT_INDEX_KEYS_PER_QUERY, + )) { promises.push( this.getTextIndexKeyValues({ databaseName, tableName, connectionId, - queryOptions: nativeTextIndexQueryOptions, + queryOptions: new Map(entries), maxParts: mapKeyIndexMaxParts, partSelection: mapKeyIndexPartSelection, dateRange, From 4e27cab923768ec60c739424969db778cbc1f272 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:48:14 -0500 Subject: [PATCH 05/27] perf: read text index values in 1024-row blocks so per-key LIMITs stop early --- packages/common-utils/src/__tests__/metadata.test.ts | 5 +++++ packages/common-utils/src/core/metadata.ts | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e42e62de12..8b17d275b9 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1383,6 +1383,10 @@ describe('Metadata', () => { expect(sql).toContain('topKWeighted('); expect(sql).toContain('LIMIT 100000'); expect(sql).not.toContain('startsWith(token,'); + expect(calls[calls.length - 1][0].clickhouse_settings).toMatchObject({ + max_rows_to_read: '0', + max_block_size: '1024', + }); }); it("routes LogAttributes['requestId'] through getMapTextIndexKeyValues (map KV text index path)", async () => { @@ -1427,6 +1431,7 @@ describe('Metadata', () => { } expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); + expect(call.clickhouse_settings.max_block_size).toBe('1024'); }); it.each([ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 90276fe1dd..63be98719c 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1828,6 +1828,9 @@ export class Metadata { // The per-key LIMIT bounds the read. A row cap would stop the // sorted dictionary read partway and leave later keys empty. max_rows_to_read: '0', + // LIMIT stops a part's read only after a whole block, so the + // default 65k-row block makes every key read far past its LIMIT. + max_block_size: '1024', }, abort_signal: signal, }) @@ -1910,6 +1913,7 @@ export class Metadata { ...this.getClickHouseSettings(), // See getMapTextIndexKeyValues. max_rows_to_read: '0', + max_block_size: '1024', }, abort_signal: signal, }) From 92b1ff0e97765caabe434f8b01fa9c663a0366c1 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:13:53 -0500 Subject: [PATCH 06/27] Revert "perf: read text index values in 1024-row blocks so per-key LIMITs stop early" This reverts commit 969a5e04b69a11f23a9b5d925df2cd032a01201f. --- packages/common-utils/src/__tests__/metadata.test.ts | 5 ----- packages/common-utils/src/core/metadata.ts | 4 ---- 2 files changed, 9 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 8b17d275b9..e42e62de12 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1383,10 +1383,6 @@ describe('Metadata', () => { expect(sql).toContain('topKWeighted('); expect(sql).toContain('LIMIT 100000'); expect(sql).not.toContain('startsWith(token,'); - expect(calls[calls.length - 1][0].clickhouse_settings).toMatchObject({ - max_rows_to_read: '0', - max_block_size: '1024', - }); }); it("routes LogAttributes['requestId'] through getMapTextIndexKeyValues (map KV text index path)", async () => { @@ -1431,7 +1427,6 @@ describe('Metadata', () => { } expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); - expect(call.clickhouse_settings.max_block_size).toBe('1024'); }); it.each([ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 63be98719c..90276fe1dd 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1828,9 +1828,6 @@ export class Metadata { // The per-key LIMIT bounds the read. A row cap would stop the // sorted dictionary read partway and leave later keys empty. max_rows_to_read: '0', - // LIMIT stops a part's read only after a whole block, so the - // default 65k-row block makes every key read far past its LIMIT. - max_block_size: '1024', }, abort_signal: signal, }) @@ -1913,7 +1910,6 @@ export class Metadata { ...this.getClickHouseSettings(), // See getMapTextIndexKeyValues. max_rows_to_read: '0', - max_block_size: '1024', }, abort_signal: signal, }) From 2eb23489d32366e70b16d46eec3cea1ba9c663ff Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:15:23 -0500 Subject: [PATCH 07/27] Revert "perf: read text index values per key with a token cap" This reverts commit e66388dafe66fd7008bcd1863f0104fdc9219874. --- .changeset/topk-text-index-values.md | 18 ++--- .../src/__tests__/metadata.int.test.ts | 16 +--- .../src/__tests__/metadata.test.ts | 36 ++------- packages/common-utils/src/core/metadata.ts | 75 ++++++++++--------- 4 files changed, 55 insertions(+), 90 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index a25ccb0f78..95f8e695d3 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,16 +2,12 @@ '@hyperdx/common-utils': patch --- -perf: read each key's text index values separately, and show the most -frequent ones +perf: show the most frequent text index values, in batches of 10 keys "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) read every token of every requested key, capped at -`max_rows_to_read` for the whole query. One key with a value per row (request -IDs, UUIDs) slowed the whole query down, and because the index is read in -sorted order, the cap left keys that sort after it with no values. Each key now -has its own query branch that reads at most 100,000 tokens, and there is no -whole-query row cap. Values come from `topKWeighted(limit)` weighted by each -token's row count, so a capped list holds the most common values, most common -first. For keys with more distinct values than the token cap, they are the most -common among the alphabetically first ones. +indexes on plain columns) used `groupUniqArray(limit)`, which keeps the +alphabetically first tokens it reads. They now use `topKWeighted(limit)` +weighted by each token's row count, so a capped list holds the most common +values, most common first. These lookups also run in parallel batches of at +most 10 keys instead of 100, so a key with very many values delays only its own +batch. diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 82778f25e5..25ff349cf5 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -925,21 +925,13 @@ describe('Metadata Integration Tests', () => { const result = await metadata.getAllKeyValues({ ...commonArgs, - keyExpressions: [ - `ResourceAttributes['${freqKey}']`, - `ResourceAttributes['${mapKey}']`, - ], + keyExpressions: [`ResourceAttributes['${freqKey}']`], maxValuesPerKey: 1, }); - // UNION ALL branches can arrive in any order. - expect(result).toHaveLength(2); - expect(result).toEqual( - expect.arrayContaining([ - { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, - { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] }, - ]), - ); + expect(result).toEqual([ + { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, + ]); }); it('routes TraceId through getTextIndexKeyValues', async () => { diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e42e62de12..bf140ba037 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1381,7 +1381,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - expect(sql).toContain('LIMIT 100000'); + expect(sql).toContain('GROUP BY key'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1400,33 +1400,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('startsWith(token,'); expect(sql).toContain('substring(token, position(token,'); - expect(sql).toContain("startsWith(token, 'requestId=')"); - }); - - it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => { - setupDefaultLogsSchema(); - - await metadata.getAllKeyValues({ - ...baseArgs, - keyExpressions: [ - "LogAttributes['requestId']", - "LogAttributes['region']", - ], - }); - - const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find( - ([opts]) => (opts.query as string).includes('startsWith(token,'), - )![0]; - const branches = (call.query as string).split('UNION ALL'); - expect(branches).toHaveLength(2); - expect(branches[0]).toContain("startsWith(token, 'requestId=')"); - expect(branches[0]).toContain("'requestId' AS key"); - expect(branches[1]).toContain("startsWith(token, 'region=')"); - for (const branch of branches) { - expect(branch).toContain('LIMIT 100000'); - } - expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); - expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); + expect(sql).toContain('GROUP BY column, key'); }); it.each([ @@ -1548,11 +1522,11 @@ describe('Metadata', () => { ), }); - const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls + const keysPerQuery = (mockClickhouseClient.query as jest.Mock).mock.calls .map(([opts]) => opts.query as string) .filter(q => q.includes('startsWith(token,')) - .map(q => q.split('UNION ALL').length); - expect(branchCounts).toEqual([10, 10, 5]); + .map(q => q.split('startsWith(token,').length - 1); + expect(keysPerQuery).toEqual([10, 10, 5]); }); it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 90276fe1dd..4f30860f18 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -78,10 +78,6 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; -// Tokens a text index values lookup reads per key. The dictionary is sorted, -// so a key with more distinct values than this gets the most frequent among -// its alphabetically first ones. Keep it above "Load more"'s 10,000 values. -const TEXT_INDEX_TOKENS_PER_KEY = 100_000; // Keys per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; @@ -1783,6 +1779,19 @@ export class Metadata { try { const sqlBranches: Array = []; for (const [columnName, info] of queryOptions.entries()) { + const orChain = concatChSql( + ' OR ', + // Inline keys as SQL-escaped literals, not bind params: ~100 + // per-key params exceed the web client's URL param budget and + // silently switch the request to a multipart body that proxies + // may reject. Keys are ingest-controlled, hence SqlString.escape. + info.keys.map( + k => + chSql`startsWith(token, ${{ + UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`), + }})`, + ), + ); const partsFilter = await this.partsOverlapFilter({ databaseName, tableName, @@ -1792,29 +1801,20 @@ export class Metadata { partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; - // One branch per key, each with its own LIMIT: the read for a key - // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a - // value per row can't slow down or starve the others. `cardinality` - // is the token's row count in the part, so topKWeighted keeps the - // most frequent values among the tokens read. - for (const k of info.keys) { - // Inline keys as SQL-escaped literals, not bind params: ~100 - // per-key params exceed the web client's URL param budget and - // silently switch the request to a multipart body that proxies - // may reject. Keys are ingest-controlled, hence SqlString.escape. - sqlBranches.push(chSql` - SELECT ${{ String: columnName }} AS column, - ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value - FROM ( - SELECT ${valueSql} AS v, cardinality + // `cardinality` is the token's row count in the part, so this keeps + // each key's most frequent values rather than the first ones read. + const sql = chSql` + SELECT * FROM ( + SELECT ${{ String: columnName }} as column, + substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} - AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}) - AND v != '' - LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} - )`); - } + AND (${orChain}) + AND ${valueSql} != '' + GROUP BY column, key + )`; + sqlBranches.push(sql); } const sql = concatChSql(' UNION ALL ', sqlBranches); @@ -1824,10 +1824,12 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', ...this.getClickHouseSettings(), - // The per-key LIMIT bounds the read. A row cap would stop the - // sorted dictionary read partway and leave later keys empty. - max_rows_to_read: '0', }, abort_signal: signal, }) @@ -1886,16 +1888,14 @@ export class Metadata { maxParts: maxParts ?? undefined, partSelection, }); - // See getMapTextIndexKeyValues for the per-column LIMIT. const sql = chSql` - SELECT ${{ String: columnName }} AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value - FROM ( - SELECT token, cardinality + SELECT * FROM ( + SELECT ${{ String: columnName }} AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND token != '' - LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} + GROUP BY key )`; sqlBranches.push(sql); } @@ -1907,9 +1907,12 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', ...this.getClickHouseSettings(), - // See getMapTextIndexKeyValues. - max_rows_to_read: '0', }, abort_signal: signal, }) From b941e4e583c6d0eaa0460ab6d98e0b551b84fc4d Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:30:13 -0500 Subject: [PATCH 08/27] Reapply "perf: read text index values per key with a token cap" This reverts commit 6525149b281abf2773a59667dcc69c153dc3cc00. --- .changeset/topk-text-index-values.md | 18 +++-- .../src/__tests__/metadata.int.test.ts | 16 +++- .../src/__tests__/metadata.test.ts | 36 +++++++-- packages/common-utils/src/core/metadata.ts | 75 +++++++++---------- 4 files changed, 90 insertions(+), 55 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 95f8e695d3..a25ccb0f78 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,12 +2,16 @@ '@hyperdx/common-utils': patch --- -perf: show the most frequent text index values, in batches of 10 keys +perf: read each key's text index values separately, and show the most +frequent ones "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) used `groupUniqArray(limit)`, which keeps the -alphabetically first tokens it reads. They now use `topKWeighted(limit)` -weighted by each token's row count, so a capped list holds the most common -values, most common first. These lookups also run in parallel batches of at -most 10 keys instead of 100, so a key with very many values delays only its own -batch. +indexes on plain columns) read every token of every requested key, capped at +`max_rows_to_read` for the whole query. One key with a value per row (request +IDs, UUIDs) slowed the whole query down, and because the index is read in +sorted order, the cap left keys that sort after it with no values. Each key now +has its own query branch that reads at most 100,000 tokens, and there is no +whole-query row cap. Values come from `topKWeighted(limit)` weighted by each +token's row count, so a capped list holds the most common values, most common +first. For keys with more distinct values than the token cap, they are the most +common among the alphabetically first ones. diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 25ff349cf5..82778f25e5 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -925,13 +925,21 @@ describe('Metadata Integration Tests', () => { const result = await metadata.getAllKeyValues({ ...commonArgs, - keyExpressions: [`ResourceAttributes['${freqKey}']`], + keyExpressions: [ + `ResourceAttributes['${freqKey}']`, + `ResourceAttributes['${mapKey}']`, + ], maxValuesPerKey: 1, }); - expect(result).toEqual([ - { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, - ]); + // UNION ALL branches can arrive in any order. + expect(result).toHaveLength(2); + expect(result).toEqual( + expect.arrayContaining([ + { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] }, + { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] }, + ]), + ); }); it('routes TraceId through getTextIndexKeyValues', async () => { diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index bf140ba037..e42e62de12 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1381,7 +1381,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - expect(sql).toContain('GROUP BY key'); + expect(sql).toContain('LIMIT 100000'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1400,7 +1400,33 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('startsWith(token,'); expect(sql).toContain('substring(token, position(token,'); - expect(sql).toContain('GROUP BY column, key'); + expect(sql).toContain("startsWith(token, 'requestId=')"); + }); + + it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['region']", + ], + }); + + const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find( + ([opts]) => (opts.query as string).includes('startsWith(token,'), + )![0]; + const branches = (call.query as string).split('UNION ALL'); + expect(branches).toHaveLength(2); + expect(branches[0]).toContain("startsWith(token, 'requestId=')"); + expect(branches[0]).toContain("'requestId' AS key"); + expect(branches[1]).toContain("startsWith(token, 'region=')"); + for (const branch of branches) { + expect(branch).toContain('LIMIT 100000'); + } + expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); + expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); }); it.each([ @@ -1522,11 +1548,11 @@ describe('Metadata', () => { ), }); - const keysPerQuery = (mockClickhouseClient.query as jest.Mock).mock.calls + const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls .map(([opts]) => opts.query as string) .filter(q => q.includes('startsWith(token,')) - .map(q => q.split('startsWith(token,').length - 1); - expect(keysPerQuery).toEqual([10, 10, 5]); + .map(q => q.split('UNION ALL').length); + expect(branchCounts).toEqual([10, 10, 5]); }); it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 4f30860f18..90276fe1dd 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -78,6 +78,10 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; +// Tokens a text index values lookup reads per key. The dictionary is sorted, +// so a key with more distinct values than this gets the most frequent among +// its alphabetically first ones. Keep it above "Load more"'s 10,000 values. +const TEXT_INDEX_TOKENS_PER_KEY = 100_000; // Keys per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; @@ -1779,19 +1783,6 @@ export class Metadata { try { const sqlBranches: Array = []; for (const [columnName, info] of queryOptions.entries()) { - const orChain = concatChSql( - ' OR ', - // Inline keys as SQL-escaped literals, not bind params: ~100 - // per-key params exceed the web client's URL param budget and - // silently switch the request to a multipart body that proxies - // may reject. Keys are ingest-controlled, hence SqlString.escape. - info.keys.map( - k => - chSql`startsWith(token, ${{ - UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`), - }})`, - ), - ); const partsFilter = await this.partsOverlapFilter({ databaseName, tableName, @@ -1801,20 +1792,29 @@ export class Metadata { partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; - // `cardinality` is the token's row count in the part, so this keeps - // each key's most frequent values rather than the first ones read. - const sql = chSql` - SELECT * FROM ( - SELECT ${{ String: columnName }} as column, - substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}, cardinality) AS value + // One branch per key, each with its own LIMIT: the read for a key + // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a + // value per row can't slow down or starve the others. `cardinality` + // is the token's row count in the part, so topKWeighted keeps the + // most frequent values among the tokens read. + for (const k of info.keys) { + // Inline keys as SQL-escaped literals, not bind params: ~100 + // per-key params exceed the web client's URL param budget and + // silently switch the request to a multipart body that proxies + // may reject. Keys are ingest-controlled, hence SqlString.escape. + sqlBranches.push(chSql` + SELECT ${{ String: columnName }} AS column, + ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value + FROM ( + SELECT ${valueSql} AS v, cardinality FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} - AND (${orChain}) - AND ${valueSql} != '' - GROUP BY column, key - )`; - sqlBranches.push(sql); + AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}) + AND v != '' + LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} + )`); + } } const sql = concatChSql(' UNION ALL ', sqlBranches); @@ -1824,12 +1824,10 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // The per-key LIMIT bounds the read. A row cap would stop the + // sorted dictionary read partway and leave later keys empty. + max_rows_to_read: '0', }, abort_signal: signal, }) @@ -1888,14 +1886,16 @@ export class Metadata { maxParts: maxParts ?? undefined, partSelection, }); + // See getMapTextIndexKeyValues for the per-column LIMIT. const sql = chSql` - SELECT * FROM ( - SELECT ${{ String: columnName }} AS key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value + SELECT ${{ String: columnName }} AS key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(token, cardinality) AS value + FROM ( + SELECT token, cardinality FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND token != '' - GROUP BY key + LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} )`; sqlBranches.push(sql); } @@ -1907,12 +1907,9 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // See getMapTextIndexKeyValues. + max_rows_to_read: '0', }, abort_signal: signal, }) From efcc6c93431b6afb978fc57883c86a1b1cbf26bb Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:31:13 -0500 Subject: [PATCH 09/27] docs: describe per-key text index value batches in the changeset --- .changeset/topk-text-index-values.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index a25ccb0f78..897e03a412 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,16 +2,19 @@ '@hyperdx/common-utils': patch --- -perf: read each key's text index values separately, and show the most -frequent ones +perf: read text index values per key, in batches of 10 keys, most frequent +first "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) read every token of every requested key, capped at -`max_rows_to_read` for the whole query. One key with a value per row (request -IDs, UUIDs) slowed the whole query down, and because the index is read in -sorted order, the cap left keys that sort after it with no values. Each key now -has its own query branch that reads at most 100,000 tokens, and there is no -whole-query row cap. Values come from `topKWeighted(limit)` weighted by each -token's row count, so a capped list holds the most common values, most common -first. For keys with more distinct values than the token cap, they are the most -common among the alphabetically first ones. +indexes on plain columns) read every token of up to 100 keys in one query, +capped at `max_rows_to_read` for the whole query. One key with a value per row +(request IDs, UUIDs) slowed every other key down, and because the index is read +in sorted order, the cap left keys that sort after it with no values. These +lookups now run in parallel queries of at most 10 keys, each key in its own +branch that reads at most 100,000 tokens, with no whole-query row cap. Values +come from `topKWeighted(limit)` weighted by each token's row count, so a capped +list holds the most common values, most common first. + +Each branch reads every part's text index once, so this is fastest when the +index uses large dictionary blocks (e.g. `text_index_dictionary_block_size = +4096`). From 5b873ccb718b19ffd0ed1cc05bd19a1b2e80660b Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:36:15 -0500 Subject: [PATCH 10/27] perf: use 4096-token dictionary blocks for key=value text indexes in seed schemas --- .changeset/text-index-dictionary-block-size.md | 15 +++++++++++++++ .../schema/seed/00002_otel_logs.sql | 9 ++++++--- .../schema/seed/00005_otel_traces.sql | 7 +++++-- 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 .changeset/text-index-dictionary-block-size.md diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md new file mode 100644 index 0000000000..9801a1ac60 --- /dev/null +++ b/.changeset/text-index-dictionary-block-size.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/otel-collector': patch +--- + +perf: use 4,096-token dictionary blocks for the key=value text indexes in the +logs and traces seed schemas + +`idx_res_attr_items`, `idx_scope_attr_items` and `idx_log_attr_items` on +`otel_logs`, and `idx_res_attr_items` and `idx_span_attr_items` on +`otel_traces`, now set `dictionary_block_size = 4096` (the default is 512). +Each part's sparse index is about 8x smaller, which cuts the per-part cost of +reading filter values from these indexes about 10x on large parts. Lookups in +these indexes decode up to 8x more tokens per block. Only new installs get the +new schema; existing tables keep their indexes until they are recreated, or +the indexes are cleared and materialized again. diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 12ec3fbe37..2872a5d894 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -32,11 +32,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + -- Larger dictionary blocks make each part's sparse index smaller. Reading + -- a key's values with mergeTreeTextIndex() loads that index per part on + -- every read, so this cuts the per-part cost about 10x on large parts. + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index b3ad5b3a9e..ba3fc5fadc 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -30,9 +30,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + -- Larger dictionary blocks make each part's sparse index smaller. Reading + -- a key's values with mergeTreeTextIndex() loads that index per part on + -- every read, so this cuts the per-part cost about 10x on large parts. + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) From 18a9831bf91b035625a400a0766f9e2225196e2b Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:50:41 -0500 Subject: [PATCH 11/27] perf: read at most 20,000 text index tokens per key --- .changeset/topk-text-index-values.md | 2 +- packages/common-utils/src/__tests__/metadata.test.ts | 4 ++-- packages/common-utils/src/core/metadata.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 897e03a412..273b3af250 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -11,7 +11,7 @@ capped at `max_rows_to_read` for the whole query. One key with a value per row (request IDs, UUIDs) slowed every other key down, and because the index is read in sorted order, the cap left keys that sort after it with no values. These lookups now run in parallel queries of at most 10 keys, each key in its own -branch that reads at most 100,000 tokens, with no whole-query row cap. Values +branch that reads at most 20,000 tokens, with no whole-query row cap. Values come from `topKWeighted(limit)` weighted by each token's row count, so a capped list holds the most common values, most common first. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e42e62de12..f1ddc782e9 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1381,7 +1381,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - expect(sql).toContain('LIMIT 100000'); + expect(sql).toContain('LIMIT 20000'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1423,7 +1423,7 @@ describe('Metadata', () => { expect(branches[0]).toContain("'requestId' AS key"); expect(branches[1]).toContain("startsWith(token, 'region=')"); for (const branch of branches) { - expect(branch).toContain('LIMIT 100000'); + expect(branch).toContain('LIMIT 20000'); } expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 90276fe1dd..7675c6ee0d 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -81,7 +81,7 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; // Tokens a text index values lookup reads per key. The dictionary is sorted, // so a key with more distinct values than this gets the most frequent among // its alphabetically first ones. Keep it above "Load more"'s 10,000 values. -const TEXT_INDEX_TOKENS_PER_KEY = 100_000; +const TEXT_INDEX_TOKENS_PER_KEY = 20_000; // Keys per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; From 9fb72526bad559ba23b656c82b8a8db55eebf102 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:10:44 -0500 Subject: [PATCH 12/27] perf: use 8192-token dictionary blocks for every text index --- .../text-index-dictionary-block-size.md | 21 +++++++++------- .changeset/topk-text-index-values.md | 4 ++-- docker/clickhouse/local/init-db-e2e.sh | 20 ++++++++-------- .../schema/seed/00002_otel_logs.sql | 24 +++++++++---------- .../schema/seed/00004_hyperdx_sessions.sql | 7 ++++-- .../schema/seed/00005_otel_traces.sql | 20 ++++++++-------- .../src/__tests__/kvItemsRewrite.int.test.ts | 2 +- .../src/__tests__/metadata.int.test.ts | 8 +++---- 8 files changed, 56 insertions(+), 50 deletions(-) diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md index 9801a1ac60..da6a7e61b6 100644 --- a/.changeset/text-index-dictionary-block-size.md +++ b/.changeset/text-index-dictionary-block-size.md @@ -2,14 +2,17 @@ '@hyperdx/otel-collector': patch --- -perf: use 4,096-token dictionary blocks for the key=value text indexes in the -logs and traces seed schemas +perf: use 8,192-token dictionary blocks for every text index in the seed +schemas -`idx_res_attr_items`, `idx_scope_attr_items` and `idx_log_attr_items` on -`otel_logs`, and `idx_res_attr_items` and `idx_span_attr_items` on -`otel_traces`, now set `dictionary_block_size = 4096` (the default is 512). -Each part's sparse index is about 8x smaller, which cuts the per-part cost of -reading filter values from these indexes about 10x on large parts. Lookups in -these indexes decode up to 8x more tokens per block. Only new installs get the -new schema; existing tables keep their indexes until they are recreated, or +Every text index on `otel_logs`, `otel_traces` and `hyperdx_sessions` now sets +`dictionary_block_size = 8192` (the default is 512). Each part's sparse index +is 16x smaller. Reading filter values with `mergeTreeTextIndex()` loads that +index for every part on every read, and on a 537M-row table 4,096-token blocks +already made those reads about 10x cheaper per part, made searches for a rare +attribute value 3-7x faster, and made searches for a common value 10-30% +faster. The setting is a per-index argument rather than the table setting +`text_index_dictionary_block_size`, which ClickHouse only accepts from 26.6. + +Only new installs get the new schema. Existing tables keep their indexes until the indexes are cleared and materialized again. diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 273b3af250..1628bebb55 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -16,5 +16,5 @@ come from `topKWeighted(limit)` weighted by each token's row count, so a capped list holds the most common values, most common first. Each branch reads every part's text index once, so this is fastest when the -index uses large dictionary blocks (e.g. `text_index_dictionary_block_size = -4096`). +index uses large dictionary blocks (e.g. `dictionary_block_size = 8192`, as the +seed schemas now do). diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index c5242dc598..4ce993493b 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -40,14 +40,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)), \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)), \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)), - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -118,10 +118,10 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 2872a5d894..fc046fdcb7 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -29,18 +29,18 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))), `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))), - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), - INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - -- Larger dictionary blocks make each part's sparse index smaller. Reading - -- a key's values with mergeTreeTextIndex() loads that index per part on - -- every read, so this cuts the per-part cost about 10x on large parts. - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), - INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') + -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each + -- part's sparse index is 16x smaller, which makes reading filter values + -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index e14258ce0d..9f106e9fed 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -20,8 +20,11 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), + -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each + -- part's sparse index is 16x smaller, which makes reading filter values + -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index ba3fc5fadc..17d0404347 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -27,17 +27,17 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)), `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))), - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - -- Larger dictionary blocks make each part's sparse index smaller. Reading - -- a key's values with mergeTreeTextIndex() loads that index per part on - -- every read, so this cuts the per-part cost about 10x on large parts. - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 4096), + -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each + -- part's sparse index is 16x smaller, which makes reading filter values + -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, - INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) + INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName), dictionary_block_size = 8192) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts index 3748c8bd24..7a59955b65 100644 --- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts +++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts @@ -70,7 +70,7 @@ describe('KV items rewrite integration', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree ORDER BY (Timestamp)`, diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 82778f25e5..915703f1c3 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -697,7 +697,7 @@ describe('Metadata Integration Tests', () => { ResourceAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')) CODEC(ZSTD(1)), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) @@ -712,7 +712,7 @@ describe('Metadata Integration Tests', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) @@ -1657,14 +1657,14 @@ describe('Metadata Integration Tests', () => { { path: 'key', tableName: 'test_map_keys_key_text_index_order', - indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`, + indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, }, { path: 'kv', tableName: 'test_map_keys_kv_text_index_order', indexDefinition: `LogAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`, + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, }, ])( 'getMapKeys - $path text index ordering', From f6c614eeca638a96aa56da2bafca9fb7c1900e3a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:55:51 -0500 Subject: [PATCH 13/27] perf: read Map text index values per key only when dictionary blocks are large --- .changeset/topk-text-index-values.md | 28 +++--- packages/app/src/components/DBRowTable.tsx | 13 +-- .../src/__tests__/metadata.test.ts | 78 +++++++++++++---- .../common-utils/src/__tests__/utils.test.ts | 50 +++++++++++ packages/common-utils/src/core/metadata.ts | 86 ++++++++++++++++--- packages/common-utils/src/core/utils.ts | 35 ++++++++ 6 files changed, 240 insertions(+), 50 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 1628bebb55..f5dd4380c4 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,19 +2,21 @@ '@hyperdx/common-utils': patch --- -perf: read text index values per key, in batches of 10 keys, most frequent -first +perf: read text index values in batches of 10 keys, most frequent first "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) read every token of up to 100 keys in one query, -capped at `max_rows_to_read` for the whole query. One key with a value per row -(request IDs, UUIDs) slowed every other key down, and because the index is read -in sorted order, the cap left keys that sort after it with no values. These -lookups now run in parallel queries of at most 10 keys, each key in its own -branch that reads at most 20,000 tokens, with no whole-query row cap. Values -come from `topKWeighted(limit)` weighted by each token's row count, so a capped -list holds the most common values, most common first. +indexes on plain columns) read every token of up to 100 keys in one query, so +one key with a value per row (request IDs, UUIDs) slowed every other key down. +These lookups now run in parallel queries of at most 10 keys, and keep at most +20,000 tokens per key. Values come from `topKWeighted(limit)` weighted by each +token's row count, so a capped list holds the most common values, most common +first. -Each branch reads every part's text index once, so this is fastest when the -index uses large dictionary blocks (e.g. `dictionary_block_size = 8192`, as the -seed schemas now do). +Map key=value indexes with large dictionary blocks (4096 or more, e.g. +`dictionary_block_size = 8192` as the seed schemas now use) read each key +separately, so a high-cardinality key stops after its 20,000 tokens. Indexes with +smaller blocks, including ClickHouse's default of 512, read all keys of a batch +in one pass: every read loads each part's sparse index, which is large with small +blocks, so one read per key would multiply that cost. The block size comes from +the index definition, then the table's `text_index_dictionary_block_size`, then +the server's. diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 10e3904b7a..8336204cd0 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -28,7 +28,10 @@ import { isJSDataTypeJSONStringifiable, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; -import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { + getEngineSetting, + splitAndTrimWithBracket, +} from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, isLogSource, @@ -1551,14 +1554,6 @@ const BLOCK_COLUMN_SETTINGS = [ { column: '_block_offset', setting: 'enable_block_offset_column' }, ]; -// engine_full ends with the SETTINGS clause. -function getEngineSetting(engineFull: string, name: string) { - const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? ''; - return splitAndTrimWithBracket(settingsClause) - .map(setting => setting.split(' = ')) - .find(([key]) => key === name)?.[1]; -} - function omitsBlockColumnSetting(engineFull: string) { return BLOCK_COLUMN_SETTINGS.some( ({ setting }) => getEngineSetting(engineFull, setting) == null, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index f1ddc782e9..e46b78ac23 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1403,8 +1403,20 @@ describe('Metadata', () => { expect(sql).toContain("startsWith(token, 'requestId=')"); }); - it('reads a capped number of tokens per Map key, without a row cap that could starve later keys', async () => { + const withDictionaryBlockSize = (size: number) => + jest.spyOn(metadata, 'getSkipIndices').mockResolvedValue([ + { + name: 'idx_log_attr_items', + type: 'text', + typeFull: `text(tokenizer = 'array', dictionary_block_size = ${size})`, + expression: 'LogAttributes', + granularity: 100000000, + }, + ]); + + it('reads each Map key in its own capped branch when the index has large dictionary blocks', async () => { setupDefaultLogsSchema(); + withDictionaryBlockSize(8192); await metadata.getAllKeyValues({ ...baseArgs, @@ -1429,6 +1441,31 @@ describe('Metadata', () => { expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); }); + it('reads all Map keys in one pass when the index has small dictionary blocks', async () => { + setupDefaultLogsSchema(); + withDictionaryBlockSize(512); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "LogAttributes['region']", + ], + }); + + const call = (mockClickhouseClient.query as jest.Mock).mock.calls.find( + ([opts]) => (opts.query as string).includes('startsWith(token,'), + )![0]; + const sql = call.query as string; + expect(sql).not.toContain('UNION ALL'); + expect(sql).toContain( + "startsWith(token, 'requestId=') OR startsWith(token, 'region=')", + ); + expect(sql).toContain('LIMIT 20000 BY key'); + expect(sql).toContain('GROUP BY key'); + expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); + }); + it.each([ ['map KV', "LogAttributes['requestId']"], ['native', 'TraceId'], @@ -1537,23 +1574,32 @@ describe('Metadata', () => { }, ); - it('splits Map text index values into queries of at most 10 keys', async () => { - setupDefaultLogsSchema(); + it.each([ + [8192, [10, 10, 5]], + [512, [1, 1, 1]], + ])( + 'splits Map text index values into queries of at most 10 keys (dictionary blocks of %d)', + async (blockSize, expectedBranchCounts) => { + setupDefaultLogsSchema(); + withDictionaryBlockSize(blockSize); - await metadata.getAllKeyValues({ - ...baseArgs, - keyExpressions: Array.from( - { length: 25 }, - (_, i) => `LogAttributes['key${i}']`, - ), - }); + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: Array.from( + { length: 25 }, + (_, i) => `LogAttributes['key${i}']`, + ), + }); - const branchCounts = (mockClickhouseClient.query as jest.Mock).mock.calls - .map(([opts]) => opts.query as string) - .filter(q => q.includes('startsWith(token,')) - .map(q => q.split('UNION ALL').length); - expect(branchCounts).toEqual([10, 10, 5]); - }); + const branchCounts = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls + .map(([opts]) => opts.query as string) + .filter(q => q.includes('startsWith(token,')) + .map(q => q.split('UNION ALL').length); + expect(branchCounts).toEqual(expectedBranchCounts); + }, + ); it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 739c5627b6..477a342d6d 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -15,6 +15,7 @@ import { getDistributedTableArgs, getFirstOrderingItem, getPercentileAggFnLevel, + getTextIndexDictionaryBlockSize, hasNonEmptyOrderBy, hasPositiveSeriesLimit, isFirstOrderByAscending, @@ -3716,4 +3717,53 @@ describe('utils', () => { expect(resolvePercentileAggFn(select)).toBe(select); }); }); + + describe('getTextIndexDictionaryBlockSize', () => { + const engineFull = + 'MergeTree ORDER BY Timestamp SETTINGS index_granularity = 8192, text_index_dictionary_block_size = 4096'; + + it.each([ + [ + 'the index argument first', + { + typeFull: "text(tokenizer = 'array', dictionary_block_size = 8192)", + engineFull, + serverMergeTreeSettings: new Map([ + ['text_index_dictionary_block_size', '1024'], + ]), + }, + 8192, + ], + [ + 'the table setting next', + { + typeFull: "text(tokenizer = 'array')", + engineFull, + serverMergeTreeSettings: new Map([ + ['text_index_dictionary_block_size', '1024'], + ]), + }, + 4096, + ], + [ + 'the server setting next', + { + typeFull: "text(tokenizer = 'array')", + engineFull: 'MergeTree ORDER BY Timestamp', + serverMergeTreeSettings: new Map([ + ['text_index_dictionary_block_size', '1024'], + ]), + }, + 1024, + ], + ['512 when nothing is known', {}, 512], + [ + '512 for a value that is not a positive integer', + { typeFull: "text(tokenizer = 'array', dictionary_block_size = 0)" }, + 512, + ], + ])('uses %s', (_, input, expected) => { + expect(getTextIndexDictionaryBlockSize(input)).toBe(expected); + }); + }); }); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 7675c6ee0d..821b828c79 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -59,6 +59,7 @@ import { import { getAlignedDateRange, getDistributedTableArgs, + getTextIndexDictionaryBlockSize, MetadataMVQueryOptions, objectHash, splitAndTrimWithBracket, @@ -84,6 +85,9 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; const TEXT_INDEX_TOKENS_PER_KEY = 20_000; // Keys per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; +// Smallest dictionary block size that gets one read per key; see +// getMapTextIndexKeyValues. +const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 4096; /** * The index read could not be scoped to the requested date range, so it was @@ -1758,6 +1762,31 @@ export class Metadata { }); } + // shortcut: parts written before a table's text_index_dictionary_block_size + // changed keep their old block size; good enough to pick a query shape. + private async getTextIndexDictionaryBlockSize({ + databaseName, + tableName, + connectionId, + indexName, + }: TableConnection & { indexName: string }): Promise { + const [skipIndices, tableMetadata, serverMergeTreeSettings] = + await Promise.all([ + this.getSkipIndices({ databaseName, tableName, connectionId }).catch( + () => [] as SkipIndexMetadata[], + ), + this.getTableMetadata({ databaseName, tableName, connectionId }).catch( + () => undefined, + ), + this.getMergeTreeSettings({ connectionId }), + ]); + return getTextIndexDictionaryBlockSize({ + typeFull: skipIndices.find(idx => idx.name === indexName)?.typeFull, + engineFull: tableMetadata?.engine_full, + serverMergeTreeSettings, + }); + } + private async getMapTextIndexKeyValues({ databaseName, tableName, @@ -1792,16 +1821,49 @@ export class Metadata { partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; - // One branch per key, each with its own LIMIT: the read for a key - // stops after TEXT_INDEX_TOKENS_PER_KEY tokens, so a key with a - // value per row can't slow down or starve the others. `cardinality` - // is the token's row count in the part, so topKWeighted keeps the - // most frequent values among the tokens read. + // Inline keys as SQL-escaped literals, not bind params: ~100 + // per-key params exceed the web client's URL param budget and + // silently switch the request to a multipart body that proxies + // may reject. Keys are ingest-controlled, hence SqlString.escape. + const keyPrefix = (k: string) => + chSql`${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}`; + // `cardinality` is the token's row count in the part, so + // topKWeighted keeps the most frequent values among the tokens read. + // + // Every read of the index loads each part's sparse index, which has + // one entry per dictionary block. With large blocks that is cheap, so + // each key gets its own read whose LIMIT stops it early, and a key + // with a value per row can't slow down the others. With small blocks + // (ClickHouse's default 512) the sparse index is large, so all keys + // share one read; LIMIT BY bounds their values but not the read. + const blockSize = await this.getTextIndexDictionaryBlockSize({ + databaseName, + tableName, + connectionId, + indexName: info.indexName, + }); + if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) { + sqlBranches.push(chSql` + SELECT ${{ String: columnName }} AS column, + key, + topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value + FROM ( + SELECT substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, + ${valueSql} AS v, + cardinality + FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) + WHERE ${partsFilter} + AND (${concatChSql( + ' OR ', + info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`), + )}) + AND v != '' + LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} BY key + ) + GROUP BY key`); + continue; + } for (const k of info.keys) { - // Inline keys as SQL-escaped literals, not bind params: ~100 - // per-key params exceed the web client's URL param budget and - // silently switch the request to a multipart body that proxies - // may reject. Keys are ingest-controlled, hence SqlString.escape. sqlBranches.push(chSql` SELECT ${{ String: columnName }} AS column, ${{ UNSAFE_RAW_SQL: SqlString.escape(k) }} AS key, @@ -1810,7 +1872,7 @@ export class Metadata { SELECT ${valueSql} AS v, cardinality FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} - AND startsWith(token, ${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}) + AND startsWith(token, ${keyPrefix(k)}) AND v != '' LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} )`); @@ -1825,8 +1887,8 @@ export class Metadata { connectionId, clickhouse_settings: { ...this.getClickHouseSettings(), - // The per-key LIMIT bounds the read. A row cap would stop the - // sorted dictionary read partway and leave later keys empty. + // A row cap would stop the sorted dictionary read partway and + // leave the keys that sort after it with no values. max_rows_to_read: '0', }, abort_signal: signal, diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 6ce5b3caba..a582af9d01 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1464,6 +1464,41 @@ export type TextIndexTokenizer = } | { type: 'array' }; +/** Reads a setting from `system.tables.engine_full`, which ends with the SETTINGS clause. */ +export function getEngineSetting(engineFull: string, name: string) { + const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? ''; + return splitAndTrimWithBracket(settingsClause) + .map(setting => setting.split(' = ')) + .find(([key]) => key === name)?.[1]; +} + +const DEFAULT_TEXT_INDEX_DICTIONARY_BLOCK_SIZE = 512; + +/** + * A text index's dictionary block size: the index argument, else the table's + * `text_index_dictionary_block_size`, else the server's, else ClickHouse's 512. + */ +export function getTextIndexDictionaryBlockSize({ + typeFull, + engineFull, + serverMergeTreeSettings, +}: { + typeFull?: string; + engineFull?: string; + serverMergeTreeSettings?: Map; +}): number { + const value = + typeFull?.match(/\bdictionary_block_size\s*=\s*(\d+)/)?.[1] ?? + (engineFull + ? getEngineSetting(engineFull, 'text_index_dictionary_block_size') + : undefined) ?? + serverMergeTreeSettings?.get('text_index_dictionary_block_size'); + const size = Number(value); + return Number.isInteger(size) && size > 0 + ? size + : DEFAULT_TEXT_INDEX_DICTIONARY_BLOCK_SIZE; +} + /** * Parses the tokenizer and any associated tokenizer parameters from a text index type definition. * From 406572b3ffa374ed809220706ea3fa7aeef8288a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:04:13 -0500 Subject: [PATCH 14/27] perf: rank small-block text index values over every token read --- .changeset/topk-text-index-values.md | 19 +++++++++---------- .../src/__tests__/metadata.test.ts | 3 ++- packages/common-utils/src/core/metadata.ts | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index f5dd4380c4..9730d039c8 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -7,16 +7,15 @@ perf: read text index values in batches of 10 keys, most frequent first "All values" lookups that read a text index (Map key=value indexes and text indexes on plain columns) read every token of up to 100 keys in one query, so one key with a value per row (request IDs, UUIDs) slowed every other key down. -These lookups now run in parallel queries of at most 10 keys, and keep at most -20,000 tokens per key. Values come from `topKWeighted(limit)` weighted by each -token's row count, so a capped list holds the most common values, most common -first. +These lookups now run in parallel queries of at most 10 keys. Values come from +`topKWeighted(limit)` weighted by each token's row count, so a capped list holds +the most common values, most common first. Map key=value indexes with large dictionary blocks (4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use) read each key -separately, so a high-cardinality key stops after its 20,000 tokens. Indexes with -smaller blocks, including ClickHouse's default of 512, read all keys of a batch -in one pass: every read loads each part's sparse index, which is large with small -blocks, so one read per key would multiply that cost. The block size comes from -the index definition, then the table's `text_index_dictionary_block_size`, then -the server's. +separately and stop after 20,000 tokens, so a high-cardinality key can't hold up +the others. Indexes with smaller blocks, including ClickHouse's default of 512, +read all keys of a batch in one pass: every read loads each part's sparse index, +which is large with small blocks, so one read per key would multiply that cost. +The block size comes from the index definition, then the table's +`text_index_dictionary_block_size`, then the server's. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e46b78ac23..ceb1f43968 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1461,7 +1461,8 @@ describe('Metadata', () => { expect(sql).toContain( "startsWith(token, 'requestId=') OR startsWith(token, 'region=')", ); - expect(sql).toContain('LIMIT 20000 BY key'); + expect(sql).not.toContain('BY key\n'); + expect(sql).not.toMatch(/LIMIT \d+/); expect(sql).toContain('GROUP BY key'); expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); }); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 821b828c79..55b6ec8ae6 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1835,7 +1835,8 @@ export class Metadata { // each key gets its own read whose LIMIT stops it early, and a key // with a value per row can't slow down the others. With small blocks // (ClickHouse's default 512) the sparse index is large, so all keys - // share one read; LIMIT BY bounds their values but not the read. + // share one read. That read can't stop early per key, so topKWeighted + // sees every token and ranks by the full counts. const blockSize = await this.getTextIndexDictionaryBlockSize({ databaseName, tableName, @@ -1858,7 +1859,6 @@ export class Metadata { info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`), )}) AND v != '' - LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} BY key ) GROUP BY key`); continue; From 0b44541be77b2c515d272d23f95035f659a5ff41 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:41:07 -0500 Subject: [PATCH 15/27] perf: keep main's batching for small-block Map text index values --- .changeset/topk-text-index-values.md | 21 ++++---- .../src/__tests__/metadata.test.ts | 4 +- packages/common-utils/src/core/metadata.ts | 50 +++++++++++++++---- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 9730d039c8..055b3c83cb 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,20 +2,21 @@ '@hyperdx/common-utils': patch --- -perf: read text index values in batches of 10 keys, most frequent first +perf: show the most frequent text index values, and read large-block Map +indexes per key "All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) read every token of up to 100 keys in one query, so -one key with a value per row (request IDs, UUIDs) slowed every other key down. -These lookups now run in parallel queries of at most 10 keys. Values come from -`topKWeighted(limit)` weighted by each token's row count, so a capped list holds -the most common values, most common first. +indexes on plain columns) kept the alphabetically first values. Values now come +from `topKWeighted(limit)` weighted by each token's row count, so a capped list +holds the most common values, most common first. Map key=value indexes with large dictionary blocks (4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use) read each key -separately and stop after 20,000 tokens, so a high-cardinality key can't hold up -the others. Indexes with smaller blocks, including ClickHouse's default of 512, -read all keys of a batch in one pass: every read loads each part's sparse index, +separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens +per key, so a key with a value per row (request IDs, UUIDs) can't hold up the +others. Indexes with smaller blocks, including ClickHouse's default of 512, keep +one read for all keys as before: every read loads each part's sparse index, which is large with small blocks, so one read per key would multiply that cost. The block size comes from the index definition, then the table's -`text_index_dictionary_block_size`, then the server's. +`text_index_dictionary_block_size`, then the server's. Text indexes on plain +columns run in parallel queries of at most 10 columns. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index ceb1f43968..7ac2975882 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1577,9 +1577,9 @@ describe('Metadata', () => { it.each([ [8192, [10, 10, 5]], - [512, [1, 1, 1]], + [512, [1]], ])( - 'splits Map text index values into queries of at most 10 keys (dictionary blocks of %d)', + 'batches Map text index values by dictionary block size (%d)', async (blockSize, expectedBranchCounts) => { setupDefaultLogsSchema(); withDictionaryBlockSize(blockSize); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 55b6ec8ae6..7970d26b56 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -83,7 +83,8 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; // so a key with more distinct values than this gets the most frequent among // its alphabetically first ones. Keep it above "Load more"'s 10,000 values. const TEXT_INDEX_TOKENS_PER_KEY = 20_000; -// Keys per text index values query; see dispatchKeyValueQueries. +// Keys per text index values query when each key gets its own read, and +// columns per plain-column text index query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; // Smallest dictionary block size that gets one read per key; see // getMapTextIndexKeyValues. @@ -1792,6 +1793,7 @@ export class Metadata { tableName, connectionId, queryOptions, + perKey, maxParts, partSelection, dateRange, @@ -1799,6 +1801,8 @@ export class Metadata { signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; + /** One read per key instead of one per column; see the comment below. */ + perKey: boolean; /** null reads every part in the time range. */ maxParts: number | null; partSelection: MapKeyIndexPartSelection; @@ -1807,7 +1811,7 @@ export class Metadata { signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${perKey}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1837,13 +1841,7 @@ export class Metadata { // (ClickHouse's default 512) the sparse index is large, so all keys // share one read. That read can't stop early per key, so topKWeighted // sees every token and ranks by the full counts. - const blockSize = await this.getTextIndexDictionaryBlockSize({ - databaseName, - tableName, - connectionId, - indexName: info.indexName, - }); - if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) { + if (!perKey) { sqlBranches.push(chSql` SELECT ${{ String: columnName }} AS column, key, @@ -3124,9 +3122,22 @@ export class Metadata { // fire all the kv fetch queries const promises: Array> = []; - // Text index lookups run in small parallel batches, so a slow, - // high-cardinality key holds up only its own batch's values. + // Map columns whose index has large dictionary blocks read each key + // separately, in small parallel batches, so a slow, high-cardinality key + // holds up only its own batch's values. The rest share one read of all + // their keys; see getMapTextIndexKeyValues. + const sharedReadOptions: TextIndexMapColumnQueryOptions = new Map(); for (const [column, options] of mapTextIndexQueryOptions) { + const blockSize = await this.getTextIndexDictionaryBlockSize({ + databaseName, + tableName, + connectionId, + indexName: options.indexName, + }); + if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) { + sharedReadOptions.set(column, options); + continue; + } for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) { promises.push( this.getMapTextIndexKeyValues({ @@ -3134,6 +3145,7 @@ export class Metadata { tableName, connectionId, queryOptions: new Map([[column, { ...options, keys }]]), + perKey: true, maxParts: mapKeyIndexMaxParts, partSelection: mapKeyIndexPartSelection, dateRange, @@ -3143,6 +3155,22 @@ export class Metadata { ); } } + if (sharedReadOptions.size > 0) { + promises.push( + this.getMapTextIndexKeyValues({ + databaseName, + tableName, + connectionId, + queryOptions: sharedReadOptions, + perKey: false, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, + dateRange, + timestampValueExpression, + signal, + }), + ); + } for (const entries of chunk( [...nativeTextIndexQueryOptions], TEXT_INDEX_KEYS_PER_QUERY, From cca9bc07b016bb743e2a79c62620632f3f45d52a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:50:26 -0500 Subject: [PATCH 16/27] perf: keep main's query and row cap for small-block Map text index values --- .changeset/topk-text-index-values.md | 33 +++++----- .../src/__tests__/metadata.test.ts | 60 +++++++++++-------- packages/common-utils/src/core/metadata.ts | 51 +++++++++------- 3 files changed, 82 insertions(+), 62 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 055b3c83cb..99adf0b189 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -2,21 +2,22 @@ '@hyperdx/common-utils': patch --- -perf: show the most frequent text index values, and read large-block Map -indexes per key +perf: read large-block Map text index values per key, most frequent first -"All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) kept the alphabetically first values. Values now come -from `topKWeighted(limit)` weighted by each token's row count, so a capped list -holds the most common values, most common first. +"All values" lookups on Map key=value text indexes with large dictionary blocks +(4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use) +now read each key separately, in parallel queries of at most 10 keys, and stop +after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs) +can't hold up the others. Values come from `topKWeighted(limit)` weighted by +each token's row count, so a capped list holds the most common values, most +common first. -Map key=value indexes with large dictionary blocks (4096 or more, e.g. -`dictionary_block_size = 8192` as the seed schemas now use) read each key -separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens -per key, so a key with a value per row (request IDs, UUIDs) can't hold up the -others. Indexes with smaller blocks, including ClickHouse's default of 512, keep -one read for all keys as before: every read loads each part's sparse index, -which is large with small blocks, so one read per key would multiply that cost. -The block size comes from the index definition, then the table's -`text_index_dictionary_block_size`, then the server's. Text indexes on plain -columns run in parallel queries of at most 10 columns. +Indexes with smaller blocks, including ClickHouse's default of 512, keep the +previous query: one read for all keys, the alphabetically first values, and the +`max_rows_to_read` cap. Every read loads each part's sparse index, which is +large with small blocks, so one read per key would multiply that cost. The block +size comes from the index definition, then the table's +`text_index_dictionary_block_size`, then the server's. + +Text indexes on plain columns now return the most frequent values with +`topKWeighted`, in parallel queries of at most 10 columns. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 7ac2975882..66639fd0ea 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2,6 +2,7 @@ import { ColumnMeta } from '@/clickhouse'; import { ClickhouseClient } from '@/clickhouse/node'; import { COLUMNS_CACHE_TTL_MS, + DEFAULT_METADATA_MAX_ROWS_TO_READ, GET_ALL_KEY_VALUES_CHUNK_SIZE, Metadata, MetadataCache, @@ -1461,10 +1462,12 @@ describe('Metadata', () => { expect(sql).toContain( "startsWith(token, 'requestId=') OR startsWith(token, 'region=')", ); - expect(sql).not.toContain('BY key\n'); - expect(sql).not.toMatch(/LIMIT \d+/); - expect(sql).toContain('GROUP BY key'); - expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); + expect(sql).toContain('groupUniqArray('); + expect(sql).toContain('GROUP BY column, key'); + expect(call.clickhouse_settings).toMatchObject({ + max_rows_to_read: String(DEFAULT_METADATA_MAX_ROWS_TO_READ), + read_overflow_mode: 'break', + }); }); it.each([ @@ -1474,6 +1477,8 @@ describe('Metadata', () => { 'reads the %s text index without a row cap, even when one is configured', async (_, keyExpression) => { setupDefaultLogsSchema(); + // Small-block Map indexes keep the cap; see the one-pass test. + withDictionaryBlockSize(8192); jest .spyOn(metadata, 'getClickHouseSettings') .mockReturnValue({ max_rows_to_read: '1000' }); @@ -1926,27 +1931,34 @@ describe('Metadata', () => { // param dump when a test fails. const MAX_VALUES = 7777; - it('threads maxValuesPerKey into the map text-index query', async () => { - setupDefaultLogsSchema(); - - await metadata.getAllKeyValues({ - ...baseArgs, - keyExpressions: ["LogAttributes['requestId']"], - maxValuesPerKey: MAX_VALUES, - }); + it.each([ + [8192, 'topKWeighted'], + [512, 'groupUniqArray'], + ])( + 'threads maxValuesPerKey into the map text-index query (dictionary blocks of %d)', + async (blockSize, aggregate) => { + setupDefaultLogsSchema(); + withDictionaryBlockSize(blockSize); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']"], + maxValuesPerKey: MAX_VALUES, + }); - const mapTextIndexCall = ( - mockClickhouseClient.query as jest.Mock - ).mock.calls.find( - (c: any[]) => - typeof c[0].query === 'string' && - c[0].query.includes('startsWith(token,'), - ); - expect(mapTextIndexCall).toBeDefined(); - expect(mapTextIndexCall![0].query).toContain( - `topKWeighted(${MAX_VALUES})(`, - ); - }); + const mapTextIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find( + (c: any[]) => + typeof c[0].query === 'string' && + c[0].query.includes('startsWith(token,'), + ); + expect(mapTextIndexCall).toBeDefined(); + expect(mapTextIndexCall![0].query).toContain( + `${aggregate}(${MAX_VALUES})(`, + ); + }, + ); it('threads maxValuesPerKey into the native text-index query', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 7970d26b56..dc9cb23525 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1831,34 +1831,28 @@ export class Metadata { // may reject. Keys are ingest-controlled, hence SqlString.escape. const keyPrefix = (k: string) => chSql`${{ UNSAFE_RAW_SQL: SqlString.escape(`${k}${info.separator}`) }}`; - // `cardinality` is the token's row count in the part, so - // topKWeighted keeps the most frequent values among the tokens read. - // // Every read of the index loads each part's sparse index, which has // one entry per dictionary block. With large blocks that is cheap, so // each key gets its own read whose LIMIT stops it early, and a key - // with a value per row can't slow down the others. With small blocks + // with a value per row can't slow down the others; `cardinality` is + // the token's row count in the part, so topKWeighted keeps the most + // frequent values among the tokens read. With small blocks // (ClickHouse's default 512) the sparse index is large, so all keys - // share one read. That read can't stop early per key, so topKWeighted - // sees every token and ranks by the full counts. + // share one read, as on main. if (!perKey) { sqlBranches.push(chSql` - SELECT ${{ String: columnName }} AS column, - key, - topKWeighted(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(v, cardinality) AS value - FROM ( - SELECT substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, - ${valueSql} AS v, - cardinality + SELECT ${{ String: columnName }} AS column, + substring(token, 1, position(token, ${{ String: info.separator }}) - 1) AS key, + groupUniqArray(${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(info.limit, 'limit') }})(${valueSql}) AS value FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND (${concatChSql( ' OR ', info.keys.map(k => chSql`startsWith(token, ${keyPrefix(k)})`), )}) - AND v != '' - ) - GROUP BY key`); + AND ${valueSql} != '' + GROUP BY column, key + `); continue; } for (const k of info.keys) { @@ -1883,12 +1877,25 @@ export class Metadata { query: sql.sql, query_params: sql.params, connectionId, - clickhouse_settings: { - ...this.getClickHouseSettings(), - // A row cap would stop the sorted dictionary read partway and - // leave the keys that sort after it with no values. - max_rows_to_read: '0', - }, + clickhouse_settings: perKey + ? { + ...this.getClickHouseSettings(), + // The per-key LIMIT bounds each read. A row cap would stop + // the sorted dictionary read partway and leave the keys that + // sort after it with no values. + max_rows_to_read: '0', + } + : { + // shortcut: bounds the shared read, but keys that sort after + // a high-cardinality key can come back with no values; drop + // it if that matters more than the read's cost. + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', + ...this.getClickHouseSettings(), + }, abort_signal: signal, }) .then(res => From d73900d41726839fcdaae9b1bbc9aa761fb4f251 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:51:57 -0500 Subject: [PATCH 17/27] perf: batch small-block Map text index values by 10 keys --- .changeset/topk-text-index-values.md | 8 +-- .../src/__tests__/metadata.test.ts | 2 +- packages/common-utils/src/core/metadata.ts | 51 ++++++------------- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 99adf0b189..11d50a1201 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -13,9 +13,11 @@ each token's row count, so a capped list holds the most common values, most common first. Indexes with smaller blocks, including ClickHouse's default of 512, keep the -previous query: one read for all keys, the alphabetically first values, and the -`max_rows_to_read` cap. Every read loads each part's sparse index, which is -large with small blocks, so one read per key would multiply that cost. The block +previous query shape (one read for all of a query's keys, the alphabetically +first values, and the `max_rows_to_read` cap), now in parallel queries of at +most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads +each part's sparse index, which is large with small blocks, so one read per key +would multiply that cost. The block size comes from the index definition, then the table's `text_index_dictionary_block_size`, then the server's. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 66639fd0ea..a1c8ef012d 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1582,7 +1582,7 @@ describe('Metadata', () => { it.each([ [8192, [10, 10, 5]], - [512, [1]], + [512, [1, 1, 1]], ])( 'batches Map text index values by dictionary block size (%d)', async (blockSize, expectedBranchCounts) => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index dc9cb23525..afc21020fb 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -83,8 +83,7 @@ export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; // so a key with more distinct values than this gets the most frequent among // its alphabetically first ones. Keep it above "Load more"'s 10,000 values. const TEXT_INDEX_TOKENS_PER_KEY = 20_000; -// Keys per text index values query when each key gets its own read, and -// columns per plain-column text index query; see dispatchKeyValueQueries. +// Keys or columns per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; // Smallest dictionary block size that gets one read per key; see // getMapTextIndexKeyValues. @@ -1837,8 +1836,8 @@ export class Metadata { // with a value per row can't slow down the others; `cardinality` is // the token's row count in the part, so topKWeighted keeps the most // frequent values among the tokens read. With small blocks - // (ClickHouse's default 512) the sparse index is large, so all keys - // share one read, as on main. + // (ClickHouse's default 512) the sparse index is large, so the + // batch's keys share one read. if (!perKey) { sqlBranches.push(chSql` SELECT ${{ String: columnName }} AS column, @@ -3129,22 +3128,18 @@ export class Metadata { // fire all the kv fetch queries const promises: Array> = []; - // Map columns whose index has large dictionary blocks read each key - // separately, in small parallel batches, so a slow, high-cardinality key - // holds up only its own batch's values. The rest share one read of all - // their keys; see getMapTextIndexKeyValues. - const sharedReadOptions: TextIndexMapColumnQueryOptions = new Map(); + // Text index lookups run in small parallel batches, so a slow, + // high-cardinality key holds up only its own batch's values. Within a + // batch, large dictionary blocks get one read per key; see + // getMapTextIndexKeyValues. for (const [column, options] of mapTextIndexQueryOptions) { - const blockSize = await this.getTextIndexDictionaryBlockSize({ - databaseName, - tableName, - connectionId, - indexName: options.indexName, - }); - if (blockSize < PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE) { - sharedReadOptions.set(column, options); - continue; - } + const perKey = + (await this.getTextIndexDictionaryBlockSize({ + databaseName, + tableName, + connectionId, + indexName: options.indexName, + })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE; for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) { promises.push( this.getMapTextIndexKeyValues({ @@ -3152,7 +3147,7 @@ export class Metadata { tableName, connectionId, queryOptions: new Map([[column, { ...options, keys }]]), - perKey: true, + perKey, maxParts: mapKeyIndexMaxParts, partSelection: mapKeyIndexPartSelection, dateRange, @@ -3162,22 +3157,6 @@ export class Metadata { ); } } - if (sharedReadOptions.size > 0) { - promises.push( - this.getMapTextIndexKeyValues({ - databaseName, - tableName, - connectionId, - queryOptions: sharedReadOptions, - perKey: false, - maxParts: mapKeyIndexMaxParts, - partSelection: mapKeyIndexPartSelection, - dateRange, - timestampValueExpression, - signal, - }), - ); - } for (const entries of chunk( [...nativeTextIndexQueryOptions], TEXT_INDEX_KEYS_PER_QUERY, From fbedb4c0adf7613e68678922c33b9deb3f289d50 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:03:57 -0500 Subject: [PATCH 18/27] docs: fold the row cap change into the text index values changeset --- .changeset/text-index-values-no-row-cap.md | 11 ----------- .changeset/topk-text-index-values.md | 11 ++++++----- 2 files changed, 6 insertions(+), 16 deletions(-) delete mode 100644 .changeset/text-index-values-no-row-cap.md diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md deleted file mode 100644 index 103ed81b01..0000000000 --- a/.changeset/text-index-values-no-row-cap.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -'@hyperdx/common-utils': patch ---- - -fix: show values for every key in text index value lookups - -"All values" lookups that read a text index (Map key=value indexes and text -indexes on plain columns) stopped at `max_rows_to_read` with -`read_overflow_mode = 'break'`. The index is read in sorted token order, so once -a high-cardinality key used up the budget, every key that sorts after it came -back with no values and no error. These lookups no longer apply a row cap. diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 11d50a1201..86398f1f36 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -10,16 +10,17 @@ now read each key separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs) can't hold up the others. Values come from `topKWeighted(limit)` weighted by each token's row count, so a capped list holds the most common values, most -common first. +common first. These reads no longer apply `max_rows_to_read`: the index is read +in sorted token order, so the cap left every key that sorts after a +high-cardinality key with no values. Indexes with smaller blocks, including ClickHouse's default of 512, keep the previous query shape (one read for all of a query's keys, the alphabetically first values, and the `max_rows_to_read` cap), now in parallel queries of at most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads each part's sparse index, which is large with small blocks, so one read per key -would multiply that cost. The block -size comes from the index definition, then the table's -`text_index_dictionary_block_size`, then the server's. +would multiply that cost. The block size comes from the index definition, then +the table's `text_index_dictionary_block_size`, then the server's. Text indexes on plain columns now return the most frequent values with -`topKWeighted`, in parallel queries of at most 10 columns. +`topKWeighted`, in parallel queries of at most 10 columns, without the row cap. From a324f9a090e48e2967f087a80247f8632ca4276e Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:02:41 -0500 Subject: [PATCH 19/27] perf: set the text index block size as a table setting after seeding --- .../text-index-dictionary-block-size.md | 25 ++++----- .changeset/topk-text-index-values.md | 3 +- docker/clickhouse/local/init-db-e2e.sh | 24 ++++----- .../schema/seed/00002_otel_logs.sql | 18 +++---- .../schema/seed/00004_hyperdx_sessions.sql | 4 +- .../schema/seed/00005_otel_traces.sql | 14 ++--- .../src/__tests__/kvItemsRewrite.int.test.ts | 5 +- .../src/__tests__/metadata.int.test.ts | 11 ++-- packages/otel-collector/cmd/migrate/main.go | 51 +++++++++++++++++++ .../otel-collector/cmd/migrate/main_test.go | 29 +++++++++++ 10 files changed, 135 insertions(+), 49 deletions(-) diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md index da6a7e61b6..35c8d3f1a3 100644 --- a/.changeset/text-index-dictionary-block-size.md +++ b/.changeset/text-index-dictionary-block-size.md @@ -2,17 +2,18 @@ '@hyperdx/otel-collector': patch --- -perf: use 8,192-token dictionary blocks for every text index in the seed -schemas +perf: use 8,192-token text index dictionary blocks on the seeded tables -Every text index on `otel_logs`, `otel_traces` and `hyperdx_sessions` now sets -`dictionary_block_size = 8192` (the default is 512). Each part's sparse index -is 16x smaller. Reading filter values with `mergeTreeTextIndex()` loads that -index for every part on every read, and on a 537M-row table 4,096-token blocks -already made those reads about 10x cheaper per part, made searches for a rare -attribute value 3-7x faster, and made searches for a common value 10-30% -faster. The setting is a per-index argument rather than the table setting -`text_index_dictionary_block_size`, which ClickHouse only accepts from 26.6. +On ClickHouse 26.6 and later, the collector's schema migration now sets +`text_index_dictionary_block_size = 8192` (the default is 512) on `otel_logs`, +`otel_traces` and `hyperdx_sessions`, with `ALTER TABLE … MODIFY SETTING` after +the seed. Each part's text index sparse index is 16x smaller. Reading filter +values with `mergeTreeTextIndex()` loads that index for every part on every +read, and on a 537M-row table 4,096-token blocks already made those reads about +10x cheaper per part, made searches for a rare attribute value 3-7x faster, and +made searches for a common value 10-30% faster. -Only new installs get the new schema. Existing tables keep their indexes until -the indexes are cleared and materialized again. +Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token +blocks, and existing parts keep theirs until they're merged. A table that +already sets `text_index_dictionary_block_size` is left alone. ClickHouse +26.2–26.5 doesn't have the setting, so those servers keep 512-token blocks. diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 86398f1f36..7ab43daeb7 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -5,7 +5,8 @@ perf: read large-block Map text index values per key, most frequent first "All values" lookups on Map key=value text indexes with large dictionary blocks -(4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use) +(4096 or more, e.g. `text_index_dictionary_block_size = 8192`, which the +collector's migration now sets on ClickHouse 26.6+) now read each key separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs) can't hold up the others. Values come from `topKWeighted(limit)` weighted by diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index 4ce993493b..8465661103 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -40,20 +40,20 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)), \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)), \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)), - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192) + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192; -- Distributed table over the e2e logs table whose DDL declares a column -- (\`ColumnMissingFromShard\`) that its underlying MergeTree table does NOT have. @@ -118,10 +118,10 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) @@ -129,7 +129,7 @@ ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + toIntervalDay(30) -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192; CREATE TABLE ${DATABASE}.e2e_hyperdx_sessions ( diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index fc046fdcb7..f555e7865d 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -32,15 +32,15 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each -- part's sparse index is 16x smaller, which makes reading filter values -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192) + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index 9f106e9fed..ac44c11f57 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each -- part's sparse index is 16x smaller, which makes reading filter values -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. - INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index 17d0404347..c019d050f0 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -30,14 +30,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each -- part's sparse index is 16x smaller, which makes reading filter values -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. - INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192), - INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192), + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, - INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName), dictionary_block_size = 8192) + INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts index 7a59955b65..fd75a190a8 100644 --- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts +++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts @@ -70,10 +70,11 @@ describe('KV items rewrite integration', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 ) ENGINE = MergeTree - ORDER BY (Timestamp)`, + ORDER BY (Timestamp) + SETTINGS text_index_dictionary_block_size = 8192`, }); await client.insert({ diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 915703f1c3..1fd29056cb 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -697,10 +697,11 @@ describe('Metadata Integration Tests', () => { ResourceAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')) CODEC(ZSTD(1)), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) + SETTINGS text_index_dictionary_block_size = 8192 `, }); @@ -712,10 +713,11 @@ describe('Metadata Integration Tests', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) + SETTINGS text_index_dictionary_block_size = 8192 `, }); }); @@ -1657,14 +1659,14 @@ describe('Metadata Integration Tests', () => { { path: 'key', tableName: 'test_map_keys_key_text_index_order', - indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, + indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`, }, { path: 'kv', tableName: 'test_map_keys_kv_text_index_order', indexDefinition: `LogAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`, }, ])( 'getMapKeys - $path text index ordering', @@ -1688,6 +1690,7 @@ describe('Metadata Integration Tests', () => { ENGINE = MergeTree() PARTITION BY toDate(Timestamp) ORDER BY Timestamp + SETTINGS text_index_dictionary_block_size = 8192 `, }); diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index b67e373099..7dcc39a84d 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "errors" "fmt" "log" "net/url" @@ -644,6 +645,49 @@ func supportsFullTextSearch(major, minor int) bool { return major > 26 || (major == 26 && minor >= 2) } +// textIndexTables are the seeded tables with text indexes. +var textIndexTables = []string{"otel_logs", "otel_traces", "hyperdx_sessions"} + +// supportsTextIndexTableSettings returns true if the ClickHouse version has the +// text_index_dictionary_block_size MergeTree setting. This requires ClickHouse +// >= 26.6. +func supportsTextIndexTableSettings(major, minor int) bool { + return major > 26 || (major == 26 && minor >= 6) +} + +// setTextIndexBlockSize gives the seeded tables 8,192-token text index +// dictionary blocks (the default is 512), so filter value reads load a 16x +// smaller sparse index per part. It runs after the seed so existing tables +// get it too; new parts use it, existing parts keep theirs until merged. A +// table that already sets the value is left alone. +func setTextIndexBlockSize(ctx context.Context, db *sql.DB, database string) error { + var firstErr error + for _, table := range textIndexTables { + var engineFull string + err := db.QueryRowContext(ctx, + "SELECT engine_full FROM system.tables WHERE database = ? AND name = ?", + database, table).Scan(&engineFull) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err == nil && strings.Contains(engineFull, "text_index_dictionary_block_size") { + continue + } + if err == nil { + _, err = db.ExecContext(ctx, fmt.Sprintf( + "ALTER TABLE `%s`.`%s` MODIFY SETTING text_index_dictionary_block_size = 8192", + database, table)) + } + if err != nil { + log.Printf("WARNING: could not set the text index block size on %s: %v", table, err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + // swapLogsSchemaForCompat replaces the full-text-search logs schema with the // compatibility variant (bloom_filter indexes) in the processed temp directory. // It removes 00002_otel_logs.sql and renames 00002_otel_logs_compat.sql to @@ -875,6 +919,13 @@ func main() { os.Exit(1) } + // Non-fatal so it can't block startup; older servers keep 512-token blocks. + if supportsTextIndexTableSettings(chMajor, chMinor) { + if err := setTextIndexBlockSize(ctx, db, cfg.Database); err != nil { + log.Printf("WARNING: text index block size: %v", err) + } + } + // CREATE TABLE IF NOT EXISTS won't update TTL on existing tables; reconcile // (opt-in) applies it to them too. Non-fatal so it can't block startup. if cfg.ReconcileTableTTL { diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 84e42687d5..605f039e70 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -958,6 +958,35 @@ func TestSupportsFullTextSearch(t *testing.T) { } } +// --------------------------------------------------------------------------- +// supportsTextIndexTableSettings +// --------------------------------------------------------------------------- + +func TestSupportsTextIndexTableSettings(t *testing.T) { + tests := []struct { + name string + major int + minor int + expected bool + }{ + {"26.6 exact threshold", 26, 6, true}, + {"26.10 above minor", 26, 10, true}, + {"27.0 above major", 27, 0, true}, + {"26.5 below minor", 26, 5, false}, + {"26.2 text indexes without the setting", 26, 2, false}, + {"25.9 below major", 25, 9, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := supportsTextIndexTableSettings(tt.major, tt.minor) + if got != tt.expected { + t.Errorf("supportsTextIndexTableSettings(%d, %d) = %v, want %v", tt.major, tt.minor, got, tt.expected) + } + }) + } +} + // --------------------------------------------------------------------------- // swapLogsSchemaForCompat // --------------------------------------------------------------------------- From 6315f7f93b14dfda4fba2e8e22589e6c9bf45650 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:18:12 -0500 Subject: [PATCH 20/27] remove unnecessary comments --- docker/otel-collector/schema/seed/00002_otel_logs.sql | 3 --- docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql | 3 --- docker/otel-collector/schema/seed/00005_otel_traces.sql | 3 --- 3 files changed, 9 deletions(-) diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index f555e7865d..12ec3fbe37 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -29,9 +29,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))), `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))), - -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each - -- part's sparse index is 16x smaller, which makes reading filter values - -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index ac44c11f57..e14258ce0d 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -20,9 +20,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each - -- part's sparse index is 16x smaller, which makes reading filter values - -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index c019d050f0..b3ad5b3a9e 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -27,9 +27,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)), `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))), - -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each - -- part's sparse index is 16x smaller, which makes reading filter values - -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts. INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), From 59a4bd17c7057b0ebed30ddafbdf4d53abeb1f2a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:28:39 -0500 Subject: [PATCH 21/27] change min dictionary block size --- packages/common-utils/src/core/metadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index afc21020fb..de029f01fd 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -87,7 +87,7 @@ const TEXT_INDEX_TOKENS_PER_KEY = 20_000; const TEXT_INDEX_KEYS_PER_QUERY = 10; // Smallest dictionary block size that gets one read per key; see // getMapTextIndexKeyValues. -const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 4096; +const PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE = 2048; /** * The index read could not be scoped to the requested date range, so it was From e0cad2425fc68942c9efbba8754bbc73d7faff26 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:41:38 -0500 Subject: [PATCH 22/27] fix: scale the text index token cap by the parts read --- .../src/__tests__/metadata.int.test.ts | 95 +++++++++++++++++++ .../src/__tests__/metadata.test.ts | 31 +++++- packages/common-utils/src/core/metadata.ts | 19 +++- 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 1fd29056cb..29bdd15cf1 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -1757,4 +1757,99 @@ describe('Metadata Integration Tests', () => { }); }, ); + + describe('getAllKeyValues - values repeated across parts', () => { + // mergeTreeTextIndex returns one row per part and token, so a token cap + // that ignores the part count stops before the later parts' values. Each + // part holds the same SHARED values plus OWN values no other part has. + const tableName = 'test_text_index_values_across_parts'; + const PARTS = 25; + const SHARED = 1000; + const OWN = 100; + const dateRange = (): [Date, Date] => [ + new Date(Date.now() - 60 * 60 * 1000), + new Date(Date.now() + 60 * 1000), + ]; + let textIndexSupported = false; + + beforeAll(async () => { + const probe = new Metadata(hdxClient, new MetadataCache()); + textIndexSupported = supportsMergeTreeTextIndex( + await probe.getServerVersion({ connectionId: 'test_connection' }), + ); + if (!textIndexSupported) return; + + await client.command({ + query: `CREATE OR REPLACE TABLE default.${tableName} ( + Timestamp DateTime64(9), + ResourceAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) MATERIALIZED + arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')), + Tags Array(String), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1, + INDEX idx_tags Tags TYPE text(tokenizer = 'array') GRANULARITY 1 + ) + ENGINE = MergeTree() + PARTITION BY toDate(Timestamp) + ORDER BY Timestamp + SETTINGS text_index_dictionary_block_size = 8192 + `, + }); + await client.command({ + query: `SYSTEM STOP MERGES default.${tableName}`, + }); + for (let i = 0; i < PARTS; i++) { + await client.command({ + query: `INSERT INTO default.${tableName} (Timestamp, ResourceAttributes, Tags) + SELECT now64(9) - INTERVAL 10 MINUTE, map('pod', v), [v] + FROM ( + SELECT if(number < ${SHARED}, concat('shared-', toString(number)), concat('own-${i}-', toString(number))) AS v + FROM numbers(${SHARED + OWN}) + )`, + }); + } + }); + + afterAll(async () => { + await client.command({ + query: `DROP TABLE IF EXISTS default.${tableName}`, + }); + }); + + it.each([ + ['Map key=value', "ResourceAttributes['pod']"], + ['plain-column', 'Tags'], + ])( + 'returns every distinct value from the %s text index, with and without a part limit', + async (_, keyExpression) => { + if (!textIndexSupported) { + console.warn( + 'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()', + ); + return; + } + + // PARTS is above the default limit of 20, so unset reads 20 parts and + // null reads all of them. + for (const [mapKeyIndexMaxParts, partsRead] of [ + [undefined, 20], + [null, PARTS], + ] as const) { + const metadata = new Metadata(hdxClient, new MetadataCache()); + const result = await metadata.getAllKeyValues({ + databaseName: 'default', + tableName, + keyExpressions: [keyExpression], + maxValuesPerKey: 10000, + mapKeyIndexMaxParts, + connectionId: 'test_connection', + dateRange: dateRange(), + timestampValueExpression: 'Timestamp', + }); + expect(result).toHaveLength(1); + expect(result[0].value).toHaveLength(SHARED + OWN * partsRead); + } + }, + ); + }); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index a1c8ef012d..70145ef44a 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1382,7 +1382,8 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - expect(sql).toContain('LIMIT 20000'); + // 20,000 tokens for each of the default 20 parts. + expect(sql).toContain('LIMIT 400000'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1436,12 +1437,38 @@ describe('Metadata', () => { expect(branches[0]).toContain("'requestId' AS key"); expect(branches[1]).toContain("startsWith(token, 'region=')"); for (const branch of branches) { - expect(branch).toContain('LIMIT 20000'); + // 20,000 tokens for each of the default 20 parts. + expect(branch).toContain('LIMIT 400000'); } expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); }); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'scales the %s text index token cap by the parts read, and drops it when every part is read', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + withDictionaryBlockSize(8192); + + for (const mapKeyIndexMaxParts of [5, null]) { + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts, + }); + } + + const tokenLimits = (mockClickhouseClient.query as jest.Mock).mock.calls + .map(([opts]) => opts.query as string) + .filter(q => q.includes('mergeTreeTextIndex(')) + .map(q => q.match(/LIMIT (\d+)/)?.[1]); + expect(tokenLimits).toEqual(['100000', undefined]); + }, + ); + it('reads all Map keys in one pass when the index has small dictionary blocks', async () => { setupDefaultLogsSchema(); withDictionaryBlockSize(512); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index de029f01fd..be0d51d792 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -79,10 +79,19 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; -// Tokens a text index values lookup reads per key. The dictionary is sorted, -// so a key with more distinct values than this gets the most frequent among -// its alphabetically first ones. Keep it above "Load more"'s 10,000 values. +// Distinct values a text index values lookup reads per key. The dictionary is +// sorted, so a key with more than this gets the most frequent among its +// alphabetically first ones. Keep it above "Load more"'s 10,000 values. const TEXT_INDEX_TOKENS_PER_KEY = 20_000; + +// mergeTreeTextIndex returns one row per part and token, so a value in every +// part read takes maxParts rows; scaling the LIMIT by maxParts keeps room for +// TEXT_INDEX_TOKENS_PER_KEY distinct values. A read of every part ("Load +// more") isn't capped. +const textIndexTokenLimit = (maxParts: number | null): ChSql => + maxParts == null + ? chSql`` + : chSql`LIMIT ${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(TEXT_INDEX_TOKENS_PER_KEY * maxParts, 'maxParts') }}`; // Keys or columns per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; // Smallest dictionary block size that gets one read per key; see @@ -1865,7 +1874,7 @@ export class Metadata { WHERE ${partsFilter} AND startsWith(token, ${keyPrefix(k)}) AND v != '' - LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} + ${textIndexTokenLimit(maxParts)} )`); } } @@ -1961,7 +1970,7 @@ export class Metadata { FROM mergeTreeTextIndex(${{ String: databaseName }}, ${{ String: tableName }}, ${{ String: info.indexName }}) WHERE ${partsFilter} AND token != '' - LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }} + ${textIndexTokenLimit(maxParts)} )`; sqlBranches.push(sql); } From 1314065c39bfcb5c555ba1f780bd55826e41d37e Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:44:36 -0500 Subject: [PATCH 23/27] feat: let operators opt out of the text index block size ALTER --- .changeset/text-index-dictionary-block-size.md | 6 ++++-- docker-compose.dev.yml | 2 ++ packages/otel-collector/cmd/migrate/main.go | 10 ++++++++-- .../otel-collector/cmd/migrate/main_test.go | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.changeset/text-index-dictionary-block-size.md b/.changeset/text-index-dictionary-block-size.md index 35c8d3f1a3..e6331b58b2 100644 --- a/.changeset/text-index-dictionary-block-size.md +++ b/.changeset/text-index-dictionary-block-size.md @@ -15,5 +15,7 @@ made searches for a common value 10-30% faster. Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token blocks, and existing parts keep theirs until they're merged. A table that -already sets `text_index_dictionary_block_size` is left alone. ClickHouse -26.2–26.5 doesn't have the setting, so those servers keep 512-token blocks. +already sets `text_index_dictionary_block_size` is left alone, and operators who +manage these tables themselves can turn the change off with +`HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false`. ClickHouse 26.2–26.5 +doesn't have the setting, so those servers keep 512-token blocks. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 335054d479..b1bba4f792 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -45,6 +45,8 @@ services: # HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL # Apply TTL changes to already-existing tables too (default false): # HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true' + # Leave text index dictionary block sizes alone (default: set 8192 on 26.6+): + # HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE: 'false' ENABLE_PROMQL: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 7dcc39a84d..6d680adf69 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -43,6 +43,9 @@ type Config struct { // Reconcile TTL on already-existing tables, not just newly created ones. ReconcileTableTTL bool + // Set 8,192-token text index dictionary blocks on the seeded tables. + SetTextIndexBlockSize bool + // TLS settings TLSCAFile string TLSCertFile string @@ -69,6 +72,7 @@ func loadConfig() (*Config, error) { MetricsTTL: getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL), SessionsTTL: getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL), ReconcileTableTTL: getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true", + SetTextIndexBlockSize: getEnv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "true") != "false", TLSCAFile: getEnv("CLICKHOUSE_TLS_CA_FILE", ""), TLSCertFile: getEnv("CLICKHOUSE_TLS_CERT_FILE", ""), TLSKeyFile: getEnv("CLICKHOUSE_TLS_KEY_FILE", ""), @@ -919,8 +923,10 @@ func main() { os.Exit(1) } - // Non-fatal so it can't block startup; older servers keep 512-token blocks. - if supportsTextIndexTableSettings(chMajor, chMinor) { + // On by default; HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false lets + // operators who manage these tables opt out. Non-fatal so it can't block + // startup; older servers keep 512-token blocks. + if cfg.SetTextIndexBlockSize && supportsTextIndexTableSettings(chMajor, chMinor) { if err := setTextIndexBlockSize(ctx, db, cfg.Database); err != nil { log.Printf("WARNING: text index block size: %v", err) } diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 605f039e70..6e2fea4620 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -111,6 +111,9 @@ func TestLoadConfig(t *testing.T) { if cfg.ReconcileTableTTL { t.Error("ReconcileTableTTL should default to false") } + if !cfg.SetTextIndexBlockSize { + t.Error("SetTextIndexBlockSize should default to true") + } if cfg.SchemaDir != dir { t.Errorf("SchemaDir: got %q, want %q", cfg.SchemaDir, dir) } @@ -629,6 +632,20 @@ func TestProcessSchemaDir_NonexistentDir(t *testing.T) { // per-signal TTL resolution + reconcile helpers // --------------------------------------------------------------------------- +func TestLoadConfig_TextIndexBlockSizeOptOut(t *testing.T) { + dir := t.TempDir() + os.Args = []string{"migrate", dir} + t.Setenv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "false") + + cfg, err := loadConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.SetTextIndexBlockSize { + t.Error("SetTextIndexBlockSize should be false") + } +} + func TestLoadConfig_PerSignalTTL(t *testing.T) { dir := t.TempDir() os.Args = []string{"migrate", dir} From 0de99662b82c61003166d8e1adaf79e51681c010 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:45:38 -0500 Subject: [PATCH 24/27] docs: say 2048 for the per-key text index threshold --- .changeset/topk-text-index-values.md | 2 +- packages/common-utils/src/__tests__/metadata.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.changeset/topk-text-index-values.md b/.changeset/topk-text-index-values.md index 7ab43daeb7..3565b3d1c8 100644 --- a/.changeset/topk-text-index-values.md +++ b/.changeset/topk-text-index-values.md @@ -5,7 +5,7 @@ perf: read large-block Map text index values per key, most frequent first "All values" lookups on Map key=value text indexes with large dictionary blocks -(4096 or more, e.g. `text_index_dictionary_block_size = 8192`, which the +(2048 or more, e.g. `text_index_dictionary_block_size = 8192`, which the collector's migration now sets on ClickHouse 26.6+) now read each key separately, in parallel queries of at most 10 keys, and stop after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 70145ef44a..e6fb60280f 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1609,6 +1609,9 @@ describe('Metadata', () => { it.each([ [8192, [10, 10, 5]], + // The per-key threshold is 2048. + [2048, [10, 10, 5]], + [1024, [1, 1, 1]], [512, [1, 1, 1]], ])( 'batches Map text index values by dictionary block size (%d)', From 94e7616bf23fd666b8da6a2525e8ad3dfbc83b7e Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:53:35 -0500 Subject: [PATCH 25/27] perf: keep the text index token cap at 20,000 outside "Load more" --- .../src/__tests__/metadata.int.test.ts | 40 ++++++++----------- .../src/__tests__/metadata.test.ts | 10 ++--- packages/common-utils/src/core/metadata.ts | 14 +++---- 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 29bdd15cf1..f5004bae7f 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -1760,8 +1760,8 @@ describe('Metadata Integration Tests', () => { describe('getAllKeyValues - values repeated across parts', () => { // mergeTreeTextIndex returns one row per part and token, so a token cap - // that ignores the part count stops before the later parts' values. Each - // part holds the same SHARED values plus OWN values no other part has. + // stops before the later parts' values. Each part holds the same SHARED + // values plus OWN values no other part has. const tableName = 'test_text_index_values_across_parts'; const PARTS = 25; const SHARED = 1000; @@ -1820,7 +1820,7 @@ describe('Metadata Integration Tests', () => { ['Map key=value', "ResourceAttributes['pod']"], ['plain-column', 'Tags'], ])( - 'returns every distinct value from the %s text index, with and without a part limit', + 'returns every distinct value from the %s text index for "Load more"', async (_, keyExpression) => { if (!textIndexSupported) { console.warn( @@ -1829,26 +1829,20 @@ describe('Metadata Integration Tests', () => { return; } - // PARTS is above the default limit of 20, so unset reads 20 parts and - // null reads all of them. - for (const [mapKeyIndexMaxParts, partsRead] of [ - [undefined, 20], - [null, PARTS], - ] as const) { - const metadata = new Metadata(hdxClient, new MetadataCache()); - const result = await metadata.getAllKeyValues({ - databaseName: 'default', - tableName, - keyExpressions: [keyExpression], - maxValuesPerKey: 10000, - mapKeyIndexMaxParts, - connectionId: 'test_connection', - dateRange: dateRange(), - timestampValueExpression: 'Timestamp', - }); - expect(result).toHaveLength(1); - expect(result[0].value).toHaveLength(SHARED + OWN * partsRead); - } + // "Load more" passes null: every part, no token cap. + const metadata = new Metadata(hdxClient, new MetadataCache()); + const result = await metadata.getAllKeyValues({ + databaseName: 'default', + tableName, + keyExpressions: [keyExpression], + maxValuesPerKey: 10000, + mapKeyIndexMaxParts: null, + connectionId: 'test_connection', + dateRange: dateRange(), + timestampValueExpression: 'Timestamp', + }); + expect(result).toHaveLength(1); + expect(result[0].value).toHaveLength(SHARED + OWN * PARTS); }, ); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e6fb60280f..185282670b 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1382,8 +1382,7 @@ describe('Metadata', () => { expect(sql).toContain('mergeTreeTextIndex('); expect(sql).toContain('topKWeighted('); - // 20,000 tokens for each of the default 20 parts. - expect(sql).toContain('LIMIT 400000'); + expect(sql).toContain('LIMIT 20000'); expect(sql).not.toContain('startsWith(token,'); }); @@ -1437,8 +1436,7 @@ describe('Metadata', () => { expect(branches[0]).toContain("'requestId' AS key"); expect(branches[1]).toContain("startsWith(token, 'region=')"); for (const branch of branches) { - // 20,000 tokens for each of the default 20 parts. - expect(branch).toContain('LIMIT 400000'); + expect(branch).toContain('LIMIT 20000'); } expect(call.clickhouse_settings.max_rows_to_read).toBe('0'); expect(call.clickhouse_settings.read_overflow_mode).toBeUndefined(); @@ -1448,7 +1446,7 @@ describe('Metadata', () => { ['map KV', "LogAttributes['requestId']"], ['native', 'TraceId'], ])( - 'scales the %s text index token cap by the parts read, and drops it when every part is read', + 'caps the %s text index read per key, and drops the cap when every part is read', async (_, keyExpression) => { setupDefaultLogsSchema(); withDictionaryBlockSize(8192); @@ -1465,7 +1463,7 @@ describe('Metadata', () => { .map(([opts]) => opts.query as string) .filter(q => q.includes('mergeTreeTextIndex(')) .map(q => q.match(/LIMIT (\d+)/)?.[1]); - expect(tokenLimits).toEqual(['100000', undefined]); + expect(tokenLimits).toEqual(['20000', undefined]); }, ); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index be0d51d792..f09979418c 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -79,19 +79,19 @@ export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; -// Distinct values a text index values lookup reads per key. The dictionary is +// Index rows a text index values lookup reads per key. The dictionary is // sorted, so a key with more than this gets the most frequent among its -// alphabetically first ones. Keep it above "Load more"'s 10,000 values. +// alphabetically first ones. const TEXT_INDEX_TOKENS_PER_KEY = 20_000; -// mergeTreeTextIndex returns one row per part and token, so a value in every -// part read takes maxParts rows; scaling the LIMIT by maxParts keeps room for -// TEXT_INDEX_TOKENS_PER_KEY distinct values. A read of every part ("Load -// more") isn't capped. +// mergeTreeTextIndex returns one row per part and token, so the cap counts a +// value once per part it's in, and values only in parts read after it are +// missed. That sample is enough for the top values of a filter, but "Load +// more" (no part limit) wants every value, so its read isn't capped. const textIndexTokenLimit = (maxParts: number | null): ChSql => maxParts == null ? chSql`` - : chSql`LIMIT ${{ UNSAFE_RAW_SQL: inlineNonNegativeInt(TEXT_INDEX_TOKENS_PER_KEY * maxParts, 'maxParts') }}`; + : chSql`LIMIT ${{ UNSAFE_RAW_SQL: String(TEXT_INDEX_TOKENS_PER_KEY) }}`; // Keys or columns per text index values query; see dispatchKeyValueQueries. const TEXT_INDEX_KEYS_PER_QUERY = 10; // Smallest dictionary block size that gets one read per key; see From b525b76fa51648656ac8db49a16de064c940b42d Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:24:57 -0500 Subject: [PATCH 26/27] perf: look up Map text index block sizes in parallel --- .../src/__tests__/metadata.test.ts | 46 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 23 ++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 185282670b..08e640d148 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -2217,6 +2217,52 @@ describe('Metadata', () => { value: ['r1'], }; + it("looks up every Map column's dictionary block size at once", async () => { + setupDefaultLogsSchema(); + const kvIndex = (mapColumn: string, indexName: string) => ({ + kv: { + columnName: mapColumn, + mapColumn, + indexName, + separator: '=', + useHasAny: false, + }, + }); + jest.spyOn(metadata, 'getMapColumnTextIndexes').mockResolvedValue( + new Map([ + ['LogAttributes', kvIndex('LogAttributes', 'idx_log_attr_items')], + [ + 'ResourceAttributes', + kvIndex('ResourceAttributes', 'idx_res_attr_items'), + ], + ]), + ); + let releaseSettings: () => void = () => {}; + const settings = jest + .spyOn(metadata, 'getMergeTreeSettings') + .mockReturnValue( + new Promise(resolve => { + releaseSettings = () => resolve(undefined); + }), + ); + + const result = metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [ + "LogAttributes['requestId']", + "ResourceAttributes['service.name']", + ], + }); + // Both lookups start before either finishes. + for (let i = 0; i < 50 && settings.mock.calls.length < 2; i++) { + await new Promise(resolve => setImmediate(resolve)); + } + expect(settings).toHaveBeenCalledTimes(2); + + releaseSettings(); + await result; + }); + it('yields each query as it finishes, without waiting for the slowest', async () => { setupDefaultLogsSchema(); let releaseMapTextIndex: () => void = () => {}; diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index f09979418c..b77e667d35 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -3141,14 +3141,21 @@ export class Metadata { // high-cardinality key holds up only its own batch's values. Within a // batch, large dictionary blocks get one read per key; see // getMapTextIndexKeyValues. - for (const [column, options] of mapTextIndexQueryOptions) { - const perKey = - (await this.getTextIndexDictionaryBlockSize({ - databaseName, - tableName, - connectionId, - indexName: options.indexName, - })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE; + // Every Map column's block size is looked up at once, not one at a time. + const mapColumns = await Promise.all( + [...mapTextIndexQueryOptions].map(async ([column, options]) => ({ + column, + options, + perKey: + (await this.getTextIndexDictionaryBlockSize({ + databaseName, + tableName, + connectionId, + indexName: options.indexName, + })) >= PER_KEY_TEXT_INDEX_MIN_DICTIONARY_BLOCK_SIZE, + })), + ); + for (const { column, options, perKey } of mapColumns) { for (const keys of chunk(options.keys, TEXT_INDEX_KEYS_PER_QUERY)) { promises.push( this.getMapTextIndexKeyValues({ From abe361f67aa3092fadd5b10f14c08e43bba69951 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:30:19 -0500 Subject: [PATCH 27/27] test: set the block size per index so fixtures run before ClickHouse 26.6 --- .../src/__tests__/kvItemsRewrite.int.test.ts | 5 ++--- .../src/__tests__/metadata.int.test.ts | 16 ++++++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts index fd75a190a8..7a59955b65 100644 --- a/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts +++ b/packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts @@ -70,11 +70,10 @@ describe('KV items rewrite integration', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree - ORDER BY (Timestamp) - SETTINGS text_index_dictionary_block_size = 8192`, + ORDER BY (Timestamp)`, }); await client.insert({ diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index f5004bae7f..5d782bc871 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -697,11 +697,10 @@ describe('Metadata Integration Tests', () => { ResourceAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')) CODEC(ZSTD(1)), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) - SETTINGS text_index_dictionary_block_size = 8192 `, }); @@ -713,11 +712,10 @@ describe('Metadata Integration Tests', () => { LogAttributeItems Array(String) MATERIALIZED arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String))) CODEC(ZSTD(1)), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY (Timestamp) - SETTINGS text_index_dictionary_block_size = 8192 `, }); }); @@ -1659,14 +1657,14 @@ describe('Metadata Integration Tests', () => { { path: 'key', tableName: 'test_map_keys_key_text_index_order', - indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`, + indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, }, { path: 'kv', tableName: 'test_map_keys_kv_text_index_order', indexDefinition: `LogAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')), - INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`, + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`, }, ])( 'getMapKeys - $path text index ordering', @@ -1690,7 +1688,6 @@ describe('Metadata Integration Tests', () => { ENGINE = MergeTree() PARTITION BY toDate(Timestamp) ORDER BY Timestamp - SETTINGS text_index_dictionary_block_size = 8192 `, }); @@ -1786,13 +1783,12 @@ describe('Metadata Integration Tests', () => { ResourceAttributeItems Array(String) MATERIALIZED arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')), Tags Array(String), - INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1, - INDEX idx_tags Tags TYPE text(tokenizer = 'array') GRANULARITY 1 + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1, + INDEX idx_tags Tags TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1 ) ENGINE = MergeTree() PARTITION BY toDate(Timestamp) ORDER BY Timestamp - SETTINGS text_index_dictionary_block_size = 8192 `, }); await client.command({