From eac4144322e8a9cfa9f3cdcd0fa8565645d78e32 Mon Sep 17 00:00:00 2001 From: ramzi Date: Thu, 12 Nov 2026 15:26:25 -0500 Subject: [PATCH 1/3] fix(app): address JSON array elements by index in row-panel filters The row panel wrote array positions inside a native JSON column as quoted keys (col.`events`.`0`.`id`). ClickHouse finds no such key, so the filter silently matched no rows. Array elements are addressed with 1-based subscripts (col.`events`[1].`id`). mergePath takes an optional set of array positions and writes [N+1] there for JSON columns. The row viewer derives the positions from the row data. Callers that pass no positions keep numeric segments as quoted keys, which facet keys from the JSON schema rely on. Co-Authored-By: Claude Sonnet 5.5 --- .changeset/json-array-paths.md | 9 ++++ packages/app/src/__tests__/utils.test.ts | 34 ++++++++++++++ .../src/components/DBRowJsonViewer.test.tsx | 45 ++++++++++++++++++- .../app/src/components/DBRowJsonViewer.tsx | 24 +++++++++- packages/app/src/utils.ts | 24 ++++++---- 5 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 .changeset/json-array-paths.md diff --git a/.changeset/json-array-paths.md b/.changeset/json-array-paths.md new file mode 100644 index 0000000000..c59cdf2124 --- /dev/null +++ b/.changeset/json-array-paths.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/app': patch +--- + +fix: address JSON array elements by index in row-panel filters + +"Add to Filters" on a value inside an array in a native JSON column now uses +1-based array subscripts, for example `events[1]`, instead of quoted keys such +as `events.0`, so the filter matches rows instead of silently returning none. diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index ee74129dcd..64a8d3e08d 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -1373,6 +1373,40 @@ describe('mergePath', () => { 'BodyJson.`service`.`name`', ); }); + + // Without array positions a numeric segment is a JSON object key, as + // before: facet keys come from the JSON schema, where `1` is a real key. + it('keeps a numeric segment as a quoted key when no array positions are given', () => { + expect(mergePath(['Body', 'events', '0', 'id'], ['Body'])).toBe( + 'Body.`events`.`0`.`id`', + ); + }); + + // The row viewer knows which segments index into an array. A JSON array + // element is addressed with a 1-based subscript; the quoted key + // (`.`0``) matches nothing, so the filter silently returns no rows. + it('emits a 1-based subscript for segments marked as array positions', () => { + expect( + mergePath( + ['message', 'events', '0', 'detail', 'parameters', '1', 'value'], + ['message'], + [], + new Set([2, 5]), + ), + ).toBe('message.`events`[1].`detail`.`parameters`[2].`value`'); + }); + + it('chains subscripts for nested arrays', () => { + expect( + mergePath(['m', 'grid', '0', '1', 'x'], ['m'], [], new Set([2, 3])), + ).toBe('m.`grid`[1][2].`x`'); + }); + + it('keeps a numeric object key quoted when it is not marked as an array position', () => { + expect(mergePath(['m', 'counts', '200', 'n'], ['m'], [], new Set())).toBe( + 'm.`counts`.`200`.`n`', + ); + }); }); describe('Map column (HDX-4369)', () => { diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index 4fc26a7a32..8408990cd5 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -3,7 +3,11 @@ import { fireEvent, screen, within } from '@testing-library/react'; import { useMaterializedAliasColumnsOption } from '@/hooks/useMaterializedAliasColumnsOption'; -import { buildJSONExtractQuery, DBRowJsonViewer } from './DBRowJsonViewer'; +import { + buildJSONExtractQuery, + DBRowJsonViewer, + getArrayIndexPositions, +} from './DBRowJsonViewer'; import { RowSidePanelContext } from './DBRowSidePanel'; // Mock Next.js router @@ -446,3 +450,42 @@ describe('DBRowJsonViewer', () => { }); }); }); + +describe('getArrayIndexPositions', () => { + const row = { + message: { + events: [ + { detail: { parameters: [{ name: 'a' }, { name: 'b', value: 'x' }] } }, + ], + counts: { '200': { n: 1 } }, + }, + }; + + it('marks the path positions whose parent is an array', () => { + expect( + Array.from( + getArrayIndexPositions(row, [ + 'message', + 'events', + '0', + 'detail', + 'parameters', + '1', + 'value', + ]), + ), + ).toEqual([2, 5]); + }); + + it('does not mark numeric keys of a plain object', () => { + expect( + Array.from( + getArrayIndexPositions(row, ['message', 'counts', '200', 'n']), + ), + ).toEqual([]); + }); + + it('returns an empty set for a missing path', () => { + expect(getArrayIndexPositions(row, ['nope', '0']).size).toBe(0); + }); +}); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index 3c3ed4852b..ca74c7f591 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -422,6 +422,23 @@ function HyperJsonMenu({ ); } +// Positions in `keyPath` whose parent value in `rowData` is an array, i.e. the +// segments that are array indexes and not object keys. +export function getArrayIndexPositions( + rowData: unknown, + keyPath: string[], +): Set { + const positions = new Set(); + let node: any = rowData; + keyPath.forEach((segment, i) => { + if (Array.isArray(node)) { + positions.add(i); + } + node = node?.[segment]; + }); + return positions; +} + export function DBRowJsonViewer({ data, jsonColumns, @@ -495,7 +512,12 @@ export function DBRowJsonViewer({ const getLineActions = useCallback( ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => { const actions: LineAction[] = []; - const fieldPath = mergePath(keyPath, jsonColumns, mapColumns); + const fieldPath = mergePath( + keyPath, + jsonColumns, + mapColumns, + getArrayIndexPositions(rowData, keyPath), + ); const isJsonColumn = keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index 7cdbf972c7..223553c064 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -1170,24 +1170,32 @@ export const formatUptime = (seconds: number) => { const escapeSqlSingleQuoted = (v: string): string => v.replace(/\\/g, '\\\\').replace(/'/g, "\\'"); +// `arrayIndexes` (optional) holds the positions in `path` that index into an +// array. It only affects JSON columns: a numeric segment at one of those +// positions is a 1-based subscript (`Col.\`events\`[1]`), not a quoted key +// (`Col.\`events\`.\`0\``), which matches nothing. Callers that cannot tell +// arrays from objects omit it and keep numeric segments as keys. export const mergePath = ( path: string[], jsonColumns: string[] = [], mapColumns: string[] = [], + arrayIndexes?: ReadonlySet, ) => { const [key, ...rest] = path; if (rest.length === 0) { return key; } if (jsonColumns.includes(key)) { - return `${key}.${rest - .map(v => - v - .split('.') - .map(v => (v.startsWith('`') && v.endsWith('`') ? v : `\`${v}\``)) - .join('.'), - ) - .join('.')}`; + return rest.reduce((expr, v, i) => { + if (arrayIndexes?.has(i + 1) && /^\d+$/.test(v)) { + return `${expr}[${Number(v) + 1}]`; + } + const quoted = v + .split('.') + .map(v => (v.startsWith('`') && v.endsWith('`') ? v : `\`${v}\``)) + .join('.'); + return `${expr}.${quoted}`; + }, key); } if (mapColumns.includes(key)) { // Map columns always take string-key subscripts; ClickHouse's Map index From 01b68882722d54cbfc46d3d373aa314f6f38cf0b Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 18 Nov 2026 13:43:22 -0500 Subject: [PATCH 2/3] fix: Tolerate hidden empty elements, filtered elements --- .../src/components/DBRowJsonViewer.test.tsx | 88 ++++++++++++++++++- .../app/src/components/DBRowJsonViewer.tsx | 40 ++++++--- packages/app/src/components/HyperJson.tsx | 6 +- .../components/__tests__/HyperJson.test.tsx | 19 ++++ 4 files changed, 139 insertions(+), 14 deletions(-) diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index 8408990cd5..4496380697 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -1,5 +1,6 @@ import React from 'react'; -import { fireEvent, screen, within } from '@testing-library/react'; +import { Provider as JotaiProvider } from 'jotai'; +import { fireEvent, screen, waitFor, within } from '@testing-library/react'; import { useMaterializedAliasColumnsOption } from '@/hooks/useMaterializedAliasColumnsOption'; @@ -274,6 +275,91 @@ describe('DBRowJsonViewer', () => { ); }); + // The rendered array index becomes a ClickHouse subscript, so it must be the + // element's real position even when the viewer has filtered the tree. + describe('JSON column arrays in a filtered tree', () => { + const mockClipboard = jest.fn(); + + beforeEach(() => { + Object.assign(navigator, { + clipboard: { writeText: mockClipboard }, + }); + }); + + afterEach(() => { + localStorage.removeItem('hdx_json_viewer_options'); + }); + + // A fresh jotai store per test keeps the stored viewer options from + // leaking between tests. + const renderJsonRow = (data: any) => + renderWithMantine( + + + + + , + ); + + const toggleHideBlanks = async (container: HTMLElement) => { + fireEvent.click(container.querySelector('[aria-haspopup="menu"]')!); + fireEvent.click(await screen.findByText('Hide blank values')); + }; + + const rowWithBlankElement = { + message: { events: [{ name: '' }, { name: 'second' }] }, + }; + + it('keeps the real index of an element after a blank one is hidden', async () => { + const { container } = renderJsonRow(rowWithBlankElement); + await toggleHideBlanks(container); + + expect(screen.getByText('[] 1 items')).toBeInTheDocument(); + clickLineButton('name', 'Add to Filters'); + expect(mockOnPropertyAddClick).toHaveBeenCalledWith( + 'toString(message.`events`[2].`name`)', + 'second', + ); + }); + + it('copies the row without nulls for hidden blank elements', async () => { + const { container } = renderJsonRow(rowWithBlankElement); + await toggleHideBlanks(container); + + fireEvent.click(screen.getByTitle('Copy row as JSON')); + expect(mockClipboard).toHaveBeenCalledWith( + JSON.stringify({ message: { events: [{ name: 'second' }] } }, null, 2), + ); + }); + + it('copies an array without nulls for hidden blank elements', async () => { + const { container } = renderJsonRow(rowWithBlankElement); + await toggleHideBlanks(container); + + clickLineButton('events', 'Copy Object'); + expect(mockClipboard).toHaveBeenCalledWith( + JSON.stringify([{ name: 'second' }], null, 2), + ); + }); + + it('uses a subscript for an array element while the property search is active', async () => { + renderJsonRow({ + message: { events: [{ name: 'first' }, { name: 'second' }] }, + }); + fireEvent.change( + screen.getByPlaceholderText('Search properties by key or value'), + { target: { value: 'second' } }, + ); + await waitFor(() => expect(screen.queryByText('first')).toBeNull()); + + clickLineButton('name', 'Add to Filters'); + expect(mockOnPropertyAddClick).toHaveBeenCalledWith( + 'toString(message.`events`[2].`name`)', + 'second', + ); + }); + }); + describe('copy functionality', () => { const mockClipboard = jest.fn(); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index ca74c7f591..f9820b31c3 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -104,15 +104,31 @@ function filterObjectRecursively(obj: any, filter: string): any { return result; } +// JSON.stringify writes array holes as `null`; drop them so copying with +// blanks hidden doesn't bring the blanks back. +function stringifyWithoutHoles(value: unknown): string { + return JSON.stringify( + value, + (_key, v) => (Array.isArray(v) ? v.filter(() => true) : v), + 2, + ); +} + function filterBlankValuesRecursively(value: any): any { if (value === null || value === '') { return undefined; } + // Leave holes instead of compacting: the rendered index becomes an array + // subscript in filters, so it must stay the element's real position. if (Array.isArray(value)) { - const filtered = value - .map(filterBlankValuesRecursively) - .filter(v => v !== undefined); + const filtered: unknown[] = []; + value.forEach((v, i) => { + const f = filterBlankValuesRecursively(v); + if (f !== undefined) { + filtered[i] = f; + } + }); return filtered.length > 0 ? filtered : undefined; } @@ -280,7 +296,7 @@ function HyperJsonMenu({ const copied = await copyTextToClipboard( typeof rowData === 'string' ? rowData - : JSON.stringify(rowData, null, 2), + : stringifyWithoutHoles(rowData), ); if (!copied) { notifications.show({ @@ -422,19 +438,20 @@ function HyperJsonMenu({ ); } -// Positions in `keyPath` whose parent value in `rowData` is an array, i.e. the -// segments that are array indexes and not object keys. +// Positions in `keyPath` whose parent value in `row` is an array, i.e. the +// segments that are array indexes and not object keys. Pass the unfiltered +// row: the property search rebuilds arrays as plain objects. export function getArrayIndexPositions( - rowData: unknown, + row: unknown, keyPath: string[], ): Set { const positions = new Set(); - let node: any = rowData; + let node: unknown = row; keyPath.forEach((segment, i) => { if (Array.isArray(node)) { positions.add(i); } - node = node?.[segment]; + node = get(node, segment); }); return positions; } @@ -516,7 +533,7 @@ export function DBRowJsonViewer({ keyPath, jsonColumns, mapColumns, - getArrayIndexPositions(rowData, keyPath), + getArrayIndexPositions(data, keyPath), ); const isJsonColumn = keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); @@ -723,7 +740,7 @@ export function DBRowJsonViewer({ } const copied = await copyTextToClipboard( - JSON.stringify(copiedObj, null, 2), + stringifyWithoutHoles(copiedObj), ); if (!copied) { notifications.show({ @@ -778,6 +795,7 @@ export function DBRowJsonViewer({ generateChartUrl, generateSearchUrl, onPropertyAddClick, + data, rowData, toggleColumn, jsonColumns, diff --git a/packages/app/src/components/HyperJson.tsx b/packages/app/src/components/HyperJson.tsx index d56a93df01..9745912125 100644 --- a/packages/app/src/components/HyperJson.tsx +++ b/packages/app/src/components/HyperJson.tsx @@ -103,10 +103,12 @@ const ValueRenderer = React.memo( ); } + // Arrays can be sparse (hidden blanks keep their index), so count present + // entries rather than `length`. if (isArray(value)) { return ( - {'[]'} {value.length} items + {'[]'} {Object.keys(value).length} items ); } @@ -218,7 +220,7 @@ const Line = React.memo( const isExpandable = React.useMemo( () => (isPlainObject(value) && Object.keys(value).length > 0) || - (isArray(value) && value.length > 0) || + (isArray(value) && Object.keys(value).length > 0) || isStringValueValidJson, [isStringValueValidJson, value], ); diff --git a/packages/app/src/components/__tests__/HyperJson.test.tsx b/packages/app/src/components/__tests__/HyperJson.test.tsx index 366d871ee4..4f32b857f7 100644 --- a/packages/app/src/components/__tests__/HyperJson.test.tsx +++ b/packages/app/src/components/__tests__/HyperJson.test.tsx @@ -126,3 +126,22 @@ describe('HyperJson key ordering', () => { expect(renderedValues(container).slice(1)).toEqual(list); }); }); + +// Hiding blank values leaves holes in arrays so each element keeps its real +// index; counts and expandability must ignore the holes. +describe('HyperJson sparse arrays', () => { + it('shows real indexes and counts only present elements', () => { + const list: string[] = []; + list[2] = 'kept'; + const { container } = renderWithMantine( + , + ); + + expect( + Array.from(container.querySelectorAll('.keyContainer > .key')).map(el => + el.textContent?.trim(), + ), + ).toEqual(['list', '2']); + expect(container).toHaveTextContent('[] 1 items'); + }); +}); From 69003b737dead985a5f0a40eda99e6df36a81302 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 18 Nov 2026 14:28:37 -0500 Subject: [PATCH 3/3] fix: Fix the array issue for parsed JSON as well --- .changeset/json-array-paths.md | 5 ++ .../src/components/DBRowJsonViewer.test.tsx | 41 ++++++++- .../app/src/components/DBRowJsonViewer.tsx | 46 ++++++++-- .../src/__tests__/jsonArrayPaths.int.test.ts | 89 +++++++++++++++++++ 4 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 packages/common-utils/src/__tests__/jsonArrayPaths.int.test.ts diff --git a/.changeset/json-array-paths.md b/.changeset/json-array-paths.md index c59cdf2124..6e34a0785a 100644 --- a/.changeset/json-array-paths.md +++ b/.changeset/json-array-paths.md @@ -7,3 +7,8 @@ fix: address JSON array elements by index in row-panel filters "Add to Filters" on a value inside an array in a native JSON column now uses 1-based array subscripts, for example `events[1]`, instead of quoted keys such as `events.0`, so the filter matches rows instead of silently returning none. + +The same applies to arrays inside a JSON string (for example a String `Body` +column), which now pass 1-based integer indexes to `JSONExtract*`. JSON strings +stored in a native JSON column are wrapped in `toString()` so `JSONExtract*` +accepts them. diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index 4496380697..81fb5b674c 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -480,7 +480,33 @@ describe('DBRowJsonViewer', () => { ['LogAttributes', 'config'], ['LogAttributes'], ), - ).toBe("JSONExtractString(LogAttributes.`config`, 'host')"); + ).toBe("JSONExtractString(toString(LogAttributes.`config`), 'host')"); + }); + + it('passes array positions in the parsed JSON as 1-based integers', () => { + expect( + buildJSONExtractQuery( + ['Body', 'events', '0', 'name'], + ['Body'], + [], + 'JSONExtractString', + [], + new Set([2]), + ), + ).toBe("JSONExtractString(Body, 'events', 1, 'name')"); + }); + + it('subscripts a JSON-column array element holding parsed JSON', () => { + expect( + buildJSONExtractQuery( + ['message', 'events', '1', 'payload', 'k'], + ['message', 'events', '1', 'payload'], + ['message'], + 'JSONExtractString', + [], + new Set([2]), + ), + ).toBe("JSONExtractString(toString(message.`events`[2].`payload`), 'k')"); }); // HDX-4369. HyperJson promotes a Map sub-value that is itself a @@ -571,6 +597,19 @@ describe('getArrayIndexPositions', () => { ).toEqual([]); }); + it('descends into string values that hold JSON', () => { + expect( + Array.from( + getArrayIndexPositions({ Body: '{"events":[{"name":"x"}]}' }, [ + 'Body', + 'events', + '0', + 'name', + ]), + ), + ).toEqual([2]); + }); + it('returns an empty set for a missing path', () => { expect(getArrayIndexPositions(row, ['nope', '0']).size).toBe(0); }); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index f9820b31c3..558b2e8ea8 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -56,6 +56,7 @@ export function buildJSONExtractQuery( jsonColumns: string[] = [], jsonExtractFn: JSONExtractFn = 'JSONExtractString', mapColumns: string[] = [], + arrayIndexes?: ReadonlySet, ): string | null { const nestedPath = keyPath.slice(parsedJsonRootPath.length); if (nestedPath.length === 0) { @@ -68,9 +69,27 @@ export function buildJSONExtractQuery( // to `isInParsedJson=true`, see HyperJson.tsx:227). Thread `mapColumns` so a // numeric-looking Map sub-key renders as `Map['1']` instead of the array // `Map[2]`. See HDX-4369. - const baseColumn = mergePath(parsedJsonRootPath, jsonColumns, mapColumns); - const jsonPathArgs = nestedPath.map(p => `'${p}'`).join(', '); - return `${jsonExtractFn}(${baseColumn}, ${jsonPathArgs})`; + const baseColumn = mergePath( + parsedJsonRootPath, + jsonColumns, + mapColumns, + arrayIndexes, + ); + // A JSON-column subpath is Dynamic, which JSONExtract* rejects. + const baseArg = + parsedJsonRootPath.length > 1 && jsonColumns.includes(parsedJsonRootPath[0]) + ? `toString(${baseColumn})` + : baseColumn; + // JSONExtract* reads an integer argument as a 1-based array index and a + // string argument as an object key, so '0' would look up a key named "0". + const jsonPathArgs = nestedPath + .map((p, i) => + arrayIndexes?.has(parsedJsonRootPath.length + i) && /^\d+$/.test(p) + ? `${Number(p) + 1}` + : `'${p}'`, + ) + .join(', '); + return `${jsonExtractFn}(${baseArg}, ${jsonPathArgs})`; } import { RowSidePanelContext } from './DBRowSidePanel'; @@ -438,9 +457,18 @@ function HyperJsonMenu({ ); } +function parseJsonString(value: string): unknown { + try { + return JSON.parse(value); + } catch { + return undefined; + } +} + // Positions in `keyPath` whose parent value in `row` is an array, i.e. the // segments that are array indexes and not object keys. Pass the unfiltered -// row: the property search rebuilds arrays as plain objects. +// row: the property search rebuilds arrays as plain objects. Descends into +// string values the viewer expands as JSON. export function getArrayIndexPositions( row: unknown, keyPath: string[], @@ -448,6 +476,9 @@ export function getArrayIndexPositions( const positions = new Set(); let node: unknown = row; keyPath.forEach((segment, i) => { + if (typeof node === 'string') { + node = parseJsonString(node); + } if (Array.isArray(node)) { positions.add(i); } @@ -529,11 +560,12 @@ export function DBRowJsonViewer({ const getLineActions = useCallback( ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => { const actions: LineAction[] = []; + const arrayIndexes = getArrayIndexPositions(data, keyPath); const fieldPath = mergePath( keyPath, jsonColumns, mapColumns, - getArrayIndexPositions(data, keyPath), + arrayIndexes, ); const isJsonColumn = keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); @@ -573,6 +605,7 @@ export function DBRowJsonViewer({ jsonColumns, jsonExtractFn, mapColumns, + arrayIndexes, ); if (jsonQuery) { filterFieldPath = jsonQuery; @@ -628,6 +661,7 @@ export function DBRowJsonViewer({ jsonColumns, jsonExtractFn, mapColumns, + arrayIndexes, ); if (jsonQuery) { @@ -673,6 +707,7 @@ export function DBRowJsonViewer({ jsonColumns, 'JSONExtractString', mapColumns, + arrayIndexes, ); if (jsonQuery) { chartFieldPath = jsonQuery; @@ -702,6 +737,7 @@ export function DBRowJsonViewer({ jsonColumns, 'JSONExtractString', mapColumns, + arrayIndexes, ); if (jsonQuery) { columnFieldPath = jsonQuery; diff --git a/packages/common-utils/src/__tests__/jsonArrayPaths.int.test.ts b/packages/common-utils/src/__tests__/jsonArrayPaths.int.test.ts new file mode 100644 index 0000000000..1e0ad6fef9 --- /dev/null +++ b/packages/common-utils/src/__tests__/jsonArrayPaths.int.test.ts @@ -0,0 +1,89 @@ +import { ClickHouseClient, createClient } from '@clickhouse/client'; + +// The row side panel (packages/app: mergePath, buildJSONExtractQuery) builds +// these expressions for values inside arrays. Its unit tests only compare SQL +// text; these check that ClickHouse accepts each form and matches the row. +describe('row panel expressions for JSON array elements', () => { + let client: ClickHouseClient; + + const DATABASE = 'default'; + const TABLE_NAME = 'json_array_paths_int_test'; + + beforeAll(async () => { + client = createClient({ + url: process.env.CLICKHOUSE_HOST || 'http://localhost:8123', + username: process.env.CLICKHOUSE_USER || 'default', + password: process.env.CLICKHOUSE_PASSWORD || '', + }); + + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE_NAME} ( + Id UInt32, + message JSON, + Body String + ) + ENGINE = MergeTree + ORDER BY Id`, + }); + + await client.insert({ + table: `${DATABASE}.${TABLE_NAME}`, + values: [ + { + Id: 1, + message: { + config: JSON.stringify({ host: 'h1' }), + events: [ + { name: 'first' }, + { name: 'second', payload: JSON.stringify({ k: 'v1' }) }, + ], + }, + Body: JSON.stringify({ events: [{ name: 'x', n: 1.5, ok: true }] }), + }, + { + Id: 2, + message: { + config: JSON.stringify({ host: 'h2' }), + events: [{ name: 'second' }, { name: 'first' }], + }, + Body: JSON.stringify({ events: [{ name: 'y', n: 2, ok: false }] }), + }, + ], + format: 'JSONEachRow', + }); + }); + + afterAll(async () => { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE_NAME}`, + }); + await client.close(); + }); + + const idsWhere = async (where: string) => { + const result = await client.query({ + query: `SELECT Id FROM ${DATABASE}.${TABLE_NAME} WHERE ${where} ORDER BY Id`, + format: 'JSONEachRow', + }); + const rows = await result.json<{ Id: number }>(); + return rows.map(r => Number(r.Id)); + }; + + it.each([ + // JSON column, value inside an array element + ["toString(message.`events`[2].`name`) = 'second'", [1]], + // String column holding JSON with an array + ["JSONExtractString(Body, 'events', 1, 'name') = 'x'", [1]], + ["JSONExtractFloat(Body, 'events', 1, 'n') = 2", [2]], + ["JSONExtractBool(Body, 'events', 1, 'ok') = true", [1]], + // JSON-column subpath holding a JSON string + ["JSONExtractString(toString(message.`config`), 'host') = 'h2'", [2]], + // JSON string under a JSON-column array element + [ + "JSONExtractString(toString(message.`events`[2].`payload`), 'k') = 'v1'", + [1], + ], + ])('%s', async (where, ids) => { + expect(await idsWhere(where)).toEqual(ids); + }); +});