Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/mv-min-date-inference.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@hyperdx/app': patch
---

feat: when a materialized view's configuration is inferred in the source form, its minimum date now defaults to the first whole bucket after the view was created. Views only hold rows inserted after they exist, so ranges before that read the source table instead of undercounting. Clear the date if the view was backfilled.
43 changes: 43 additions & 0 deletions packages/app/src/utils/__tests__/materializedViews.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,49 @@ describe('inferMaterializedViewConfig', () => {
});
});

it.each([
{
createdAt: '2026-11-20T04:12:18Z',
minDate: '2026-11-20T05:00:00.000Z',
},
// Created on a bucket boundary: that bucket is already whole
{
createdAt: '2026-11-20T04:00:00Z',
minDate: '2026-11-20T04:00:00.000Z',
},
])(
'should start a view created at $createdAt at its first whole bucket',
async ({ createdAt, minDate }) => {
const getTableMetadata = mockMetadata.getTableMetadata;
mockMetadata.getTableMetadata = jest.fn().mockImplementation(args =>
args.tableName === 'test_mv'
? Promise.resolve({
...mv.meta,
metadata_modification_time: createdAt,
})
: getTableMetadata(args),
);

const actualConfig = await inferMaterializedViewConfig(
{
databaseName: 'test_db',
tableName: 'test_mv',
connectionId: 'test_connection',
},
{
databaseName: 'test_db',
tableName: 'test_source_table',
connectionId: 'test_connection',
},
mockMetadata,
);

// One-hour buckets, from the view's toStartOfHour
expect(actualConfig?.minGranularity).toBe('1 hour');
expect(actualConfig?.minDate).toBe(minDate);
},
);

it('should infer materialized view configuration when given the name of a SummingMergeTree target table', async () => {
const sourceTableConnection: TableConnection = {
databaseName: 'test_db',
Expand Down
30 changes: 30 additions & 0 deletions packages/app/src/utils/materializedViews.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ import {
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
getAlignedDateRange,
Granularity,
splitAndTrimWithBracket,
} from '@hyperdx/common-utils/dist/core/utils';
import {
InternalAggregateFunction,
MaterializedViewConfiguration,
SQLIntervalSchema,
} from '@hyperdx/common-utils/dist/types';

export const MV_AGGREGATE_FUNCTIONS = [
Expand Down Expand Up @@ -456,12 +458,40 @@ export async function inferMaterializedViewConfig(
.map(col => col.name)
.join(', ');

const minDate = inferMinDate(mvMetadata, minGranularity);

return {
databaseName: mvTableMetadata.database,
tableName: mvTableMetadata.name,
dimensionColumns,
minGranularity,
timestampColumn,
aggregatedColumns,
...(minDate != null ? { minDate } : {}),
};
}

/**
* The earliest time a materialized view holds complete data for: the start of
* the first whole bucket after the view was created. A view only captures rows
* inserted after it exists, so treating it as complete for earlier ranges would
* undercount them. A backfilled view covers more; the date can be cleared in
* the source form then.
*/
function inferMinDate(
mvMetadata: TableMetadata | undefined,
minGranularity: string,
): string | undefined {
const createdAt = mvMetadata?.metadata_modification_time
? new Date(mvMetadata.metadata_modification_time)
: undefined;
if (createdAt == null || Number.isNaN(createdAt.getTime())) {
return undefined;
}

const granularity = SQLIntervalSchema.safeParse(minGranularity);
const firstWholeBucket = granularity.success
? getAlignedDateRange([createdAt, createdAt], granularity.data)[1]
: createdAt;
return firstWholeBucket.toISOString();
}
Loading