From eb4e0921c3a67ff0c12febcc887c2341265b8bae Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:09:31 +0000 Subject: [PATCH] feat(app): infer a materialized view's minimum date from when it was created A materialized view only captures rows inserted after it exists. Without a minimum date the optimizer treats it as complete for any range, so charts over earlier ranges read a view that started empty and undercounted. Inferring the configuration in the source form now sets the minimum date to the first whole bucket after the view's creation time, so those ranges read the source table. Co-Authored-By: Claude Opus 5.5 --- .changeset/mv-min-date-inference.md | 5 +++ .../utils/__tests__/materializedViews.test.ts | 43 +++++++++++++++++++ packages/app/src/utils/materializedViews.ts | 30 +++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 .changeset/mv-min-date-inference.md diff --git a/.changeset/mv-min-date-inference.md b/.changeset/mv-min-date-inference.md new file mode 100644 index 0000000000..6689e18696 --- /dev/null +++ b/.changeset/mv-min-date-inference.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +feat: when a materialized view's configuration is inferred in the source form, its minimum date now defaults to the first whole bucket after the view was created. Views only hold rows inserted after they exist, so ranges before that read the source table instead of undercounting. Clear the date if the view was backfilled. diff --git a/packages/app/src/utils/__tests__/materializedViews.test.ts b/packages/app/src/utils/__tests__/materializedViews.test.ts index d2b8003309..e5435ce35e 100644 --- a/packages/app/src/utils/__tests__/materializedViews.test.ts +++ b/packages/app/src/utils/__tests__/materializedViews.test.ts @@ -300,6 +300,49 @@ describe('inferMaterializedViewConfig', () => { }); }); + it.each([ + { + createdAt: '2026-11-20T04:12:18Z', + minDate: '2026-11-20T05:00:00.000Z', + }, + // Created on a bucket boundary: that bucket is already whole + { + createdAt: '2026-11-20T04:00:00Z', + minDate: '2026-11-20T04:00:00.000Z', + }, + ])( + 'should start a view created at $createdAt at its first whole bucket', + async ({ createdAt, minDate }) => { + const getTableMetadata = mockMetadata.getTableMetadata; + mockMetadata.getTableMetadata = jest.fn().mockImplementation(args => + args.tableName === 'test_mv' + ? Promise.resolve({ + ...mv.meta, + metadata_modification_time: createdAt, + }) + : getTableMetadata(args), + ); + + const actualConfig = await inferMaterializedViewConfig( + { + databaseName: 'test_db', + tableName: 'test_mv', + connectionId: 'test_connection', + }, + { + databaseName: 'test_db', + tableName: 'test_source_table', + connectionId: 'test_connection', + }, + mockMetadata, + ); + + // One-hour buckets, from the view's toStartOfHour + expect(actualConfig?.minGranularity).toBe('1 hour'); + expect(actualConfig?.minDate).toBe(minDate); + }, + ); + it('should infer materialized view configuration when given the name of a SummingMergeTree target table', async () => { const sourceTableConnection: TableConnection = { databaseName: 'test_db', diff --git a/packages/app/src/utils/materializedViews.ts b/packages/app/src/utils/materializedViews.ts index 5fb79d37da..07360575bf 100644 --- a/packages/app/src/utils/materializedViews.ts +++ b/packages/app/src/utils/materializedViews.ts @@ -10,12 +10,14 @@ import { TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; import { + getAlignedDateRange, Granularity, splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; import { InternalAggregateFunction, MaterializedViewConfiguration, + SQLIntervalSchema, } from '@hyperdx/common-utils/dist/types'; export const MV_AGGREGATE_FUNCTIONS = [ @@ -456,6 +458,8 @@ export async function inferMaterializedViewConfig( .map(col => col.name) .join(', '); + const minDate = inferMinDate(mvMetadata, minGranularity); + return { databaseName: mvTableMetadata.database, tableName: mvTableMetadata.name, @@ -463,5 +467,31 @@ export async function inferMaterializedViewConfig( minGranularity, timestampColumn, aggregatedColumns, + ...(minDate != null ? { minDate } : {}), }; } + +/** + * The earliest time a materialized view holds complete data for: the start of + * the first whole bucket after the view was created. A view only captures rows + * inserted after it exists, so treating it as complete for earlier ranges would + * undercount them. A backfilled view covers more; the date can be cleared in + * the source form then. + */ +function inferMinDate( + mvMetadata: TableMetadata | undefined, + minGranularity: string, +): string | undefined { + const createdAt = mvMetadata?.metadata_modification_time + ? new Date(mvMetadata.metadata_modification_time) + : undefined; + if (createdAt == null || Number.isNaN(createdAt.getTime())) { + return undefined; + } + + const granularity = SQLIntervalSchema.safeParse(minGranularity); + const firstWholeBucket = granularity.success + ? getAlignedDateRange([createdAt, createdAt], granularity.data)[1] + : createdAt; + return firstWholeBucket.toISOString(); +}