diff --git a/.changeset/emerging-signals-one-tree.md b/.changeset/emerging-signals-one-tree.md new file mode 100644 index 000000000..54b804cc8 --- /dev/null +++ b/.changeset/emerging-signals-one-tree.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +--- + +fix: MCP `emerging_signals` mines both windows in one Drain tree, in a fixed +order (baseline first), and compares them by cluster, so a token that is +constant in one window and varies in the other no longer splits a pattern into +one "new" and one "gone". New messages that Drain merges into a baseline pattern +are reported as `new_variant`, so they aren't hidden by the merge. The baseline +window excludes its end, so a row at the boundary is counted once. diff --git a/packages/api/src/mcp/__tests__/emergingSignalsOrder.test.ts b/packages/api/src/mcp/__tests__/emergingSignalsOrder.test.ts new file mode 100644 index 000000000..26c6d0ab3 --- /dev/null +++ b/packages/api/src/mcp/__tests__/emergingSignalsOrder.test.ts @@ -0,0 +1,389 @@ +// Unit test: emerging_signals mines both windows into one Drain tree in a +// fixed order, so the result doesn't depend on which window's query finishes +// first. Mocks fetchWindowSample, so no ClickHouse or Mongo is needed. + +const mockFetchWindowSample = jest.fn(); + +jest.mock('@/mcp/tools/query/runEventPatterns', () => { + const actual = jest.requireActual('@/mcp/tools/query/runEventPatterns'); + return { + ...actual, + fetchWindowSample: (...args: unknown[]) => mockFetchWindowSample(...args), + }; +}); +jest.mock('@/models/source', () => ({})); +jest.mock('@/controllers/sources', () => ({})); +jest.mock('@/controllers/connection', () => ({})); + +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +import { registerEmergingSignals } from '@/mcp/tools/query/emergingSignals'; +import type { McpContext, RegisterToolFn, ToolResult } from '@/mcp/tools/types'; + +type Handler = Parameters[2]; + +function buildHandler(): Handler { + let captured: Handler | undefined; + const registerTool: RegisterToolFn = (_name, _config, handler) => { + captured = handler; + }; + const context: McpContext = { teamId: 'team-1', userId: 'user-1' }; + const server = new McpServer({ name: 'test', version: '0.0.0' }); + registerEmergingSignals({ server, context, registerTool }); + if (!captured) throw new Error('handler was not registered'); + return captured; +} + +function textOf(result: ToolResult): string { + const item = result.content[0]; + if (!item || item.type !== 'text') throw new Error('expected text content'); + return item.text; +} + +const lines = (day: string, account: string, n: number) => + Array.from( + { length: n }, + (_, i) => + `${day} 06:${String(i % 60).padStart(2, '0')}:12 ERROR Sync failed for account ${account}: HTTP 500`, + ); +const repeat = (line: string, n: number) => + Array.from({ length: n }, () => line); + +function sample(rows: string[], startDate: Date, endDate: Date) { + return { + sampleRows: rows.map(body => ({ + __hdx_pattern_body: body, + __hdx_pattern_ts: startDate.toISOString(), + })), + totalCount: rows.length, + bodyColumn: 'Body', + startDate, + endDate, + }; +} + +type Pattern = { + pattern: string; + status: string; + sample: string; + baselinePattern?: string; + variantShare?: number; +}; + +const DAY_BEFORE = { + baselineStart: '2026-11-03T00:00:00Z', + baselineEnd: '2026-11-04T00:00:00Z', + currentStart: '2026-11-04T00:00:00Z', + currentEnd: '2026-11-05T00:00:00Z', +}; + +async function run( + baselineRows: string[], + currentRows: string[], + slowWindow: 'baseline' | 'current' = 'baseline', + windows = DAY_BEFORE, +): Promise<{ emerging: Pattern[]; disappeared: Pattern[] }> { + mockFetchWindowSample.mockImplementation( + async ( + _teamId: string, + _sourceId: string, + startDate: Date, + endDate: Date, + ) => { + const isBaseline = + startDate.getTime() === new Date(windows.baselineStart).getTime(); + if ((slowWindow === 'baseline') === isBaseline) { + await new Promise(resolve => setTimeout(resolve, 20)); + } + return sample( + isBaseline ? baselineRows : currentRows, + startDate, + endDate, + ); + }, + ); + const result = await buildHandler()({ + sourceId: '000000000000000000000001', + currentStartTime: windows.currentStart, + currentEndTime: windows.currentEnd, + baselineStartTime: windows.baselineStart, + baselineEndTime: windows.baselineEnd, + // The handler is called directly, so pass the schema defaults. + sampleSize: 10_000, + topN: 20, + minShareRatio: 3, + }); + return JSON.parse(textOf(result)); +} + +const allPatterns = (out: { emerging: Pattern[]; disappeared: Pattern[] }) => [ + ...out.emerging, + ...out.disappeared, +]; + +describe('clickstack_emerging_signals with one Drain tree', () => { + beforeEach(() => jest.clearAllMocks()); + + it('gives the same result whichever window query finishes first', async () => { + // Insertion order matters here: the account is constant in the baseline + // and different in the current window. + const baseline = lines('2026-11-03', 'acme', 20); + const current = lines('2026-11-04', 'globex', 20); + const baselineLast = await run(baseline, current, 'baseline'); + const currentLast = await run(baseline, current, 'current'); + expect(baselineLast).toEqual(currentLast); + expect(baselineLast.emerging.map(p => p.status)).toEqual(['new_variant']); + }); + + it('makes the earlier window half-open, whichever it is', async () => { + const exclusiveEnd = () => + mockFetchWindowSample.mock.calls + .filter(call => call[4]?.endExclusive === true) + .map(call => String(call[3]?.toISOString())); + await run(['a'], ['b'], 'current'); + expect(exclusiveEnd()).toEqual(['2026-11-04T00:00:00.000Z']); + mockFetchWindowSample.mockClear(); + // Baseline after the current window: the current window is the earlier one. + await run(['a'], ['b'], 'current', { + baselineStart: '2026-11-05T00:00:00Z', + baselineEnd: '2026-11-06T00:00:00Z', + currentStart: '2026-11-04T00:00:00Z', + currentEnd: '2026-11-05T00:00:00Z', + }); + expect(exclusiveEnd()).toEqual(['2026-11-05T00:00:00.000Z']); + }); + + it('treats the same dated line in both windows as one pattern', async () => { + const out = await run( + [ + ...lines('2026-11-03', 'acme', 20), + ...repeat('cache warmed in 3 regions', 10), + ], + [ + ...lines('2026-11-04', 'acme', 20), + ...repeat('disk almost full on volume data', 10), + ], + ); + const patterns = allPatterns(out).map(p => p.pattern); + expect(patterns.some(p => p.includes('Sync failed'))).toBe(false); + expect(patterns).toEqual( + expect.arrayContaining([ + expect.stringContaining('disk almost full'), + expect.stringContaining('cache warmed'), + ]), + ); + }); + + it('reports a new message that merged into a similar baseline pattern', async () => { + const out = await run(repeat('user login failed: invalid password', 20), [ + ...repeat('user login failed: invalid password', 10), + ...repeat('user login failed: account locked', 10), + ]); + expect(out.emerging).toEqual([ + expect.objectContaining({ + status: 'new_variant', + baselinePattern: 'user login failed: invalid password', + sample: 'user login failed: account locked', + }), + ]); + expect(out.disappeared).toEqual([]); + }); + + it('reports a word that was constant in the baseline and changed now', async () => { + // A different account name in the same message is a real change: the new + // message is a new_variant, and the old one is gone. + const out = await run( + lines('2026-11-03', 'acme', 20), + lines('2026-11-04', 'globex', 20), + ); + expect(out.emerging).toEqual([ + expect.objectContaining({ + status: 'new_variant', + sample: expect.stringContaining('account globex'), + }), + ]); + expect(out.disappeared).toEqual([ + expect.objectContaining({ + status: 'gone', + sample: expect.stringContaining('account acme'), + }), + ]); + }); + + it('reports new values of a field that had one value in the baseline', async () => { + // Twenty baseline rows all said host=db-1, so new hosts are a real change. + const out = await run( + repeat('connect failed host=db-1 retry later', 20), + Array.from( + { length: 20 }, + (_, i) => `connect failed host=db-${i} retry later`, + ), + ); + expect(out.emerging).toEqual([ + expect.objectContaining({ + status: 'new_variant', + baselinePattern: 'connect failed host=db-1 retry later', + }), + ]); + }); + + it('reports a new status code in an established pattern', async () => { + const out = await run(repeat('GET /api/cart 200', 20), [ + ...repeat('GET /api/cart 200', 14), + ...repeat('GET /api/cart 500', 6), + ]); + expect(out.emerging).toEqual([ + expect.objectContaining({ + status: 'new_variant', + sample: 'GET /api/cart 500', + variantShare: 0.3, + }), + ]); + expect(out.disappeared).toEqual([]); + }); + + it('gives one cluster at most one emerging entry', async () => { + // The baseline message also triples while a variant appears: only the + // variant is listed in emerging. + const out = await run( + [ + ...repeat('user login failed: invalid password', 10), + ...repeat('cache warmed in 3 regions', 90), + ], + [ + ...repeat('user login failed: invalid password', 30), + ...repeat('user login failed: account locked', 10), + ...repeat('cache warmed in 3 regions', 60), + ], + ); + const login = out.emerging.filter(p => p.pattern.startsWith('user login')); + expect(login).toEqual([expect.objectContaining({ status: 'new_variant' })]); + }); + + it('treats klog and Spark dated lines on two days as one pattern', async () => { + const out = await run( + [ + ...repeat('I1103 06:12:12.345678 1 main.go:42] sync ok', 20), + ...repeat('26/11/03 06:12:12 INFO Executor: task done', 20), + ], + [ + ...repeat('I1104 06:12:12.345678 1 main.go:42] sync ok', 20), + ...repeat('26/11/04 06:12:12 INFO Executor: task done', 20), + ], + ); + expect(out.emerging).toEqual([]); + expect(out.disappeared).toEqual([]); + }); + + it('does not report a one-row baseline pattern whose number changed', async () => { + const out = await run( + ['request took 123 ms', ...repeat('cache warmed in 3 regions', 19)], + [ + 'request took 456 ms', + 'request took 789 ms', + ...repeat('cache warmed in 3 regions', 18), + ], + ); + expect(out.emerging).toEqual([]); + expect(out.disappeared).toEqual([]); + }); + + it('reports the old message as disappeared when a variant replaced it', async () => { + const out = await run(repeat('user login failed: invalid password', 20), [ + ...repeat('user login failed: account locked', 10), + ...repeat('cache warmed in 3 regions', 10), + ]); + expect(out.emerging).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: 'new_variant', + sample: 'user login failed: account locked', + }), + ]), + ); + expect(out.disappeared).toEqual([ + expect.objectContaining({ + status: 'gone', + pattern: 'user login failed: invalid password', + sample: 'user login failed: invalid password', + }), + ]); + }); + + it('treats ctime-style dated lines on two days as one pattern', async () => { + const out = await run( + repeat('Tue Nov 3 06:12:12 2026 job finished state ok', 20), + repeat('Wed Nov 4 06:12:12 2026 job finished state ok', 20), + ); + expect(out.emerging).toEqual([]); + expect(out.disappeared).toEqual([]); + }); + + it('counts only the rows Drain put in the cluster', async () => { + // The longer line shares the prefix but has more tokens, so Drain keeps it + // in its own cluster: it is a new pattern, not part of the variant. + const out = await run(repeat('user login failed: invalid password', 20), [ + ...repeat('user login failed: invalid password', 10), + ...repeat('user login failed: account locked', 10), + ...repeat('user login failed: token expired for tenant 42', 10), + ]); + const variant = out.emerging.find(p => p.status === 'new_variant'); + expect(variant).toMatchObject({ + baselinePattern: 'user login failed: invalid password', + sample: 'user login failed: account locked', + variantShare: 0.3333, + }); + expect(out.emerging).toContainEqual( + expect.objectContaining({ + status: 'new', + pattern: expect.stringContaining('token expired for tenant'), + }), + ); + }); + + it('does not report lines that fit a template the baseline generalized', async () => { + const out = await run( + [ + ...repeat('job alpha finished ok', 10), + ...repeat('job beta finished ok', 10), + ], + repeat('job gamma finished ok', 20), + ); + expect(out.emerging).toEqual([]); + expect(out.disappeared).toEqual([]); + }); + + it('labels a disappeared pattern by its shift, not new_variant', async () => { + // One sampled variant row is below the new-pattern floor, so the old + // pattern is simply rarer now. + const out = await run(repeat('user login failed: invalid password', 20), [ + 'user login failed: account locked', + ...repeat('cache warmed in 3 regions', 19), + ]); + expect(out.disappeared).toEqual([ + expect.objectContaining({ status: 'shifted' }), + ]); + expect(out.disappeared[0]).not.toHaveProperty('baselinePattern'); + expect(allPatterns(out).some(p => p.status === 'new_variant')).toBe(false); + }); + + it('handles an empty window on either side', async () => { + const rows = repeat('user login failed: invalid password', 10); + const noCurrent = await run(rows, []); + expect(noCurrent.emerging).toEqual([]); + expect(noCurrent.disappeared.map(p => p.status)).toEqual(['gone']); + const noBaseline = await run([], rows); + expect(noBaseline.emerging.map(p => p.status)).toEqual(['new']); + expect(noBaseline.disappeared).toEqual([]); + }); + + it('reports nothing for identical windows', async () => { + const rows = [ + ...repeat('user login failed: invalid password', 10), + ...repeat('cache warmed in 3 regions', 10), + ]; + const out = await run(rows, rows); + expect(out.emerging).toEqual([]); + expect(out.disappeared).toEqual([]); + }); +}); diff --git a/packages/api/src/mcp/tools/query/emergingSignals.ts b/packages/api/src/mcp/tools/query/emergingSignals.ts index be00d9637..61621d88c 100644 --- a/packages/api/src/mcp/tools/query/emergingSignals.ts +++ b/packages/api/src/mcp/tools/query/emergingSignals.ts @@ -1,3 +1,8 @@ +import { + flattenBody, + TemplateMiner, + TemplateMinerConfig, +} from '@hyperdx/common-utils/dist/drain'; import { z } from 'zod'; import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; @@ -6,7 +11,7 @@ import { mcpUserError } from '@/mcp/utils/errors'; import { type Reduction, trimToolResponse } from '@/utils/trimToolResponse'; import { parseTimeRange } from './helpers'; -import { mineWindowPatterns, normalizeTemplate } from './runEventPatterns'; +import { fetchWindowSample, mineWindowSample } from './runEventPatterns'; import { sourceIdSchema, whereLanguageSchema, whereSchema } from './schemas'; // ─── Classification ──────────────────────────────────────────────────────── @@ -156,7 +161,8 @@ export function registerEmergingSignals({ 'event patterns (Drain) in BOTH windows and set-differences them:\n' + ' - emerging: patterns whose share of the window is >= minShareRatio× ' + 'higher now than in baseline (includes brand-new templates absent from ' + - 'baseline)\n' + + 'baseline, and new messages that merged into a baseline pattern: ' + + 'status "new_variant")\n' + ' - disappeared: patterns that were common in baseline but >= ' + 'minShareRatio× rarer (or absent) now\n\n' + 'WHY NOT clickstack_event_deltas: event_deltas compares ATTRIBUTE VALUE ' + @@ -200,34 +206,63 @@ export function registerEmergingSignals({ ); } - const opts = { + // One Drain tree for both windows: the same line lands in the same + // cluster in either window, so the windows are compared by cluster id. + // Separate trees keep tokens that were constant in one window literal + // and generalize them in the other, which splits the same pattern. + const fetchOpts = { where: input.where, whereLanguage: input.whereLanguage, bodyExpression: input.bodyExpression, sampleSize: input.sampleSize, - trendBuckets: 0, }; - - const [curRes, baseRes] = await Promise.all([ - mineWindowPatterns( + // The earlier window is half-open, so a row at the shared boundary is + // counted only once (the baseline is usually first, but may come after). + const baselineFirst = base.endDate <= cur.startDate; + const [curSample, baseSample] = await Promise.all([ + fetchWindowSample( teamId.toString(), input.sourceId, cur.startDate, cur.endDate, - opts, + { ...fetchOpts, endExclusive: !baselineFirst }, ), - mineWindowPatterns( + fetchWindowSample( teamId.toString(), input.sourceId, base.startDate, base.endDate, - opts, + { ...fetchOpts, endExclusive: baselineFirst }, ), ]); - if ('error' in curRes) return curRes.error; + if ('error' in curSample) return curSample.error; + if ('error' in baseSample) return baseSample.error; + + // Mine in a fixed order (baseline, then current): Drain depends on + // insertion order, so this keeps the result independent of which query + // finished first. + const miner = new TemplateMiner(new TemplateMinerConfig()); + const baseRes = mineWindowSample(baseSample, { trendBuckets: 0, miner }); if ('error' in baseRes) return baseRes.error; + // Each baseline cluster as it was before the current window is added + // (token copies, so later template changes don't reach them). + const baseline = new Map< + string, + { tokens: string[]; rows: number; sample: string } + >(); + for (const p of baseRes.patterns) { + const cluster = miner.drain.getCluster(Number(p.id)); + if (cluster) + baseline.set(p.id, { + tokens: [...cluster.logTemplateTokens], + rows: p.sampleCount, + sample: String(p.samples?.[0]?.__hdx_pattern_body ?? p.pattern), + }); + } + const curRes = mineWindowSample(curSample, { trendBuckets: 0, miner }); + if ('error' in curRes) return curRes.error; - // Build share-of-window maps keyed by normalized template. + // Build share-of-window maps keyed by cluster id. type Agg = { pattern: string; curShare: number; @@ -235,13 +270,15 @@ export function registerEmergingSignals({ curCount: number; baseCount: number; sample: string; + // Set on a new message that Drain merged into a baseline pattern. + variant?: { baselinePattern: string; share: number }; }; const byKey = new Map(); const ingest = (res: typeof curRes, which: 'cur' | 'base') => { const sampled = res.sampledCount || 1; for (const p of res.patterns) { - const key = normalizeTemplate(p.pattern); + const key = p.id; let a = byKey.get(key); if (!a) { a = { @@ -257,11 +294,7 @@ export function registerEmergingSignals({ byKey.set(key, a); } const share = p.sampleCount / sampled; - // Accumulate rather than overwrite: if two Drain clusters in ONE - // window normalize to the same key, both contribute their share/count - // instead of the second silently clobbering the first. Each window is - // ingested exactly once and all fields init to 0, so additive merge is - // correct for both colliding and non-colliding keys. + // Each window is ingested once and the fields start at 0. if (which === 'cur') { a.curShare += share; a.curCount += p.estimatedCount; @@ -273,6 +306,42 @@ export function registerEmergingSignals({ }; ingest(curRes, 'cur'); ingest(baseRes, 'base'); + // A cluster's template can still generalize after a window first sees + // it, so report each cluster's template from the finished tree. + for (const [key, a] of byKey) { + a.pattern = + miner.drain.getCluster(Number(key))?.getTemplate() ?? a.pattern; + } + + // One tree merges a new message into a similar baseline pattern (e.g. + // "login failed: invalid password" absorbs "login failed: account + // locked"). A current row is a new variant when Drain put it in a + // baseline cluster whose template the current window changed, and its + // masked tokens don't fit the cluster's baseline template (the same + // compare Drain's match() uses). Baseline clusters with fewer than 3 + // sampled rows are skipped: too rare to know which tokens are constant. + const changed = new Map(); + for (const [key, b] of baseline) { + const now = miner.drain.getCluster(Number(key))?.getTemplate(); + if (b.rows >= 3 && now !== undefined && now !== b.tokens.join(' ')) + changed.set(key, b.tokens); + } + const variants = new Map(); + curSample.sampleRows.forEach((row, i) => { + const key = String(curRes.rowClusterIds.at(i)); + const tmpl = changed.get(key); + if (!tmpl) return; + const raw = String(row.__hdx_pattern_body ?? ''); + const tokens = miner.tokenize(flattenBody(raw)); + if ( + tokens.length === tmpl.length && + miner.drain.getSeqDistance(tmpl, tokens, true)[0] === 1 + ) + return; + const v = variants.get(key); + if (v) v.rows++; + else variants.set(key, { rows: 1, sample: raw }); + }); const ratio = input.minShareRatio; // Minimum share for a BRAND-NEW pattern (absent from baseline) to count @@ -285,15 +354,48 @@ export function registerEmergingSignals({ curRes.sampledCount > 0 ? 2 / curRes.sampledCount : 0; const emerging: Agg[] = []; const disappeared: Agg[] = []; - for (const a of byKey.values()) { - const verdict = classifyShift(a, ratio, newPatternShareFloor); - if (verdict === 'emerging') emerging.push(a); - else if (verdict === 'disappeared') disappeared.push(a); + const sampled = curRes.sampledCount || 1; + for (const [key, a] of byKey) { + const v = variants.get(key); + // Below the floor (e.g. one sampled row) a variant is noise, like a + // one-row new pattern: classify the cluster as a whole. + if (!v || v.rows / sampled < newPatternShareFloor) { + const verdict = classifyShift(a, ratio, newPatternShareFloor); + if (verdict === 'emerging') emerging.push(a); + else if (verdict === 'disappeared') disappeared.push(a); + continue; + } + // The new message is its own emerging entry; the rest of the cluster + // (the baseline message) is listed only if it disappeared, so one + // cluster never takes two emerging slots. + const b = baseline.get(key); + const baselinePattern = (b?.tokens ?? []).join(' '); + const variantCount = Math.round(v.rows * curRes.sampleMultiplier); + emerging.push({ + pattern: a.pattern, + curShare: v.rows / sampled, + baseShare: 0, + curCount: variantCount, + baseCount: 0, + sample: v.sample, + variant: { baselinePattern, share: v.rows / sampled }, + }); + const rest = { + pattern: baselinePattern, + curShare: + Math.max(Math.round(a.curShare * sampled) - v.rows, 0) / sampled, + baseShare: a.baseShare, + curCount: Math.max(a.curCount - variantCount, 0), + baseCount: a.baseCount, + sample: b?.sample ?? baselinePattern, + }; + if (classifyShift(rest, ratio, newPatternShareFloor) === 'disappeared') + disappeared.push(rest); } - // Rank by absolute share swing. - emerging.sort( - (x, y) => y.curShare - y.baseShare - (x.curShare - x.baseShare), - ); + // Rank by absolute share swing (a variant entry has baseShare 0, so its + // swing is its own share, the same unit as a brand-new pattern). + const swing = (a: Agg) => a.curShare - a.baseShare; + emerging.sort((x, y) => swing(y) - swing(x)); disappeared.sort( (x, y) => y.baseShare - y.curShare - (x.baseShare - x.curShare), ); @@ -304,9 +406,20 @@ export function registerEmergingSignals({ baselineShare: Math.round(a.baseShare * 10000) / 10000, currentEstimatedCount: a.curCount, baselineEstimatedCount: a.baseCount, - status: - a.baseShare === 0 ? 'new' : a.curShare === 0 ? 'gone' : 'shifted', + status: a.variant + ? 'new_variant' + : a.baseShare === 0 + ? 'new' + : a.curShare === 0 + ? 'gone' + : 'shifted', sample: a.sample.slice(0, 300), + ...(a.variant + ? { + baselinePattern: a.variant.baselinePattern, + variantShare: Math.round(a.variant.share * 10000) / 10000, + } + : {}), }); const output = { @@ -341,7 +454,11 @@ export function registerEmergingSignals({ usage: 'emerging = patterns newly present or >= minShareRatio× more frequent ' + '(by share of window) now vs baseline; status "new" means absent from ' + - 'baseline entirely. disappeared = the reverse. Shares are fractions of ' + + 'baseline entirely; status "new_variant" means new messages that ' + + 'merged into a baseline pattern (baselinePattern is the old ' + + 'template, sample a new line, variantShare their share); if the ' + + 'old message dropped, it is listed in disappeared. ' + + 'disappeared = the reverse. Shares are fractions of ' + "each window's sampled rows, so they are comparable across windows of " + 'different volume. An empty emerging list means nothing novel crossed ' + 'the threshold — a valid finding.', diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index a6d6fbb50..b4dd62caa 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -6,6 +6,7 @@ import { type Granularity, splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; +import type { TemplateMiner } from '@hyperdx/common-utils/dist/drain'; import { buildPatternPredicate, DATE_MASK, @@ -53,51 +54,65 @@ const drilldownWhereFailedCounter = getCounter( // ─── Reusable window mining ────────────────────────────────────────────────── +// Shared by runEventPatterns (single window) and emerging_signals (two-window +// diff). Each returns either an error result (MCP shape) or the data. + +type WindowOptions = { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + bodyExpression?: string; + sampleSize?: number; + trendBuckets?: number; + abortSignal?: AbortSignal; + /** Dashboard variables and their selected values. Omitted outside a dashboard context. */ + variables?: ChartVariable[]; + /** Also render every row filter as one SQL condition for drill-down. */ + includeDrilldownWhere?: boolean; + /** Shared Drain miner, so cluster ids are comparable across windows. */ + miner?: TemplateMiner; + /** Exclude rows at exactly endDate (half-open window), so two adjacent windows don't share a row. */ + endExclusive?: boolean; +}; + /** - * Sample + Drain-mine one time window and return the raw pattern groups - * (template + counts), without MCP response formatting. Shared by - * runEventPatterns (single window) and emerging_signals (two-window diff). - * - * Returns either an error result (MCP shape) or the mined patterns. + * Row filters as one SQL condition ('' when there are none). Undefined when + * not requested. The source table filter is always included; the caller + * filter only when `includesUserFilter` is true (it could not be rendered as + * SQL otherwise). */ -export async function mineWindowPatterns( +type Drilldown = { where: string; includesUserFilter: boolean }; + +type WindowSample = { + sampleRows: Record[]; + totalCount: number; + bodyColumn: string; + startDate: Date; + endDate: Date; + drilldown?: Drilldown; +}; + +type MinedWindow = { + patterns: ReturnType['patterns']; + sampleMultiplier: number; + /** Drain cluster id of each sampled row, in sampleRows order. */ + rowClusterIds: number[]; + totalCount: number; + sampledCount: number; + bodyColumn: string; + trendGranularity?: Granularity; + trendBucketStarts: number[]; + drilldown?: Drilldown; +}; + +/** Sample rows and count the total for one window, without mining. */ +export async function fetchWindowSample( teamId: string, sourceId: string, startDate: Date, endDate: Date, - options?: { - where?: string; - whereLanguage?: 'lucene' | 'sql'; - bodyExpression?: string; - sampleSize?: number; - trendBuckets?: number; - abortSignal?: AbortSignal; - /** Dashboard variables and their selected values. Omitted outside a dashboard context. */ - variables?: ChartVariable[]; - /** Also render every row filter as one SQL condition for drill-down. */ - includeDrilldownWhere?: boolean; - }, -): Promise< - | { error: McpErrorResult } - | { - patterns: ReturnType['patterns']; - sampleMultiplier: number; - totalCount: number; - sampledCount: number; - bodyColumn: string; - trendGranularity?: Granularity; - trendBucketStarts: number[]; - /** - * Row filters as one SQL condition ('' when there are none). Undefined - * when not requested. The source table filter is always included; the - * caller filter only when `includesUserFilter` is true (it could not be - * rendered as SQL otherwise). - */ - drilldown?: { where: string; includesUserFilter: boolean }; - } -> { + options?: WindowOptions, +): Promise<{ error: McpErrorResult } | WindowSample> { const sampleSize = options?.sampleSize ?? 10_000; - const trendBuckets = options?.trendBuckets ?? 0; const source = await getSource(teamId, sourceId); if (!source) { @@ -200,6 +215,7 @@ export async function mineWindowPatterns( orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], limit: { limit: sampleSize, offset: 0 }, dateRange: [startDate, endDate] as [Date, Date], + ...(options?.endExclusive ? { dateRangeEndInclusive: false } : {}), variables: options?.variables, } satisfies ChartConfigWithDateRange; @@ -221,6 +237,7 @@ export async function mineWindowPatterns( useTextIndexForImplicitColumn, limit: { limit: 1, offset: 0 }, dateRange: [startDate, endDate] as [Date, Date], + ...(options?.endExclusive ? { dateRangeEndInclusive: false } : {}), variables: options?.variables, } satisfies ChartConfigWithDateRange; @@ -309,20 +326,27 @@ export async function mineWindowPatterns( where: andSql(tableFilterSql, userWhereSql ?? undefined), includesUserFilter: userWhereSql !== null, }; + return { sampleRows, totalCount, bodyColumn, startDate, endDate, drilldown }; +} - // Wrap the synchronous Drain mining in try/catch. Without this a thrown - // Drain error escapes the helper — and since callers (emerging_signals) - // await it inside Promise.all and only inspect the returned {error} union - // AFTER the await, an unwrapped throw becomes an unhandled rejection / 500 - // instead of a clean MCP error result. +/** Drain-mine a fetched window. Synchronous, so callers control the order windows enter a shared miner. */ +export function mineWindowSample( + sample: WindowSample, + options?: Pick, +): { error: McpErrorResult } | MinedWindow { + const { sampleRows, totalCount, bodyColumn, startDate, endDate, drilldown } = + sample; + // Return a Drain error as an {error} result instead of throwing, so callers + // that only inspect the union get a clean MCP error rather than a 500. let mined: ReturnType; try { mined = minePatterns(sampleRows, { totalCount, startDate, endDate, - trendBuckets, + trendBuckets: options?.trendBuckets ?? 0, maxSamples: 5, + miner: options?.miner, getBody: row => { const raw = row.__hdx_pattern_body; return raw != null ? String(raw) : ''; @@ -343,29 +367,33 @@ export async function mineWindowPatterns( drilldown, patterns: mined.patterns, sampleMultiplier: mined.sampleMultiplier, + rowClusterIds: mined.rowClusterIds, sampledCount: sampleRows.length, trendGranularity: mined.trendGranularity, trendBucketStarts: mined.trendBucketStarts, }; } -// ─── Drain template normalization (emerging_signals cross-window keying) ───── - /** - * Normalize a Drain template so the same logical pattern matches across - * windows despite Drain assigning different cluster ids each run. We collapse - * runs of whitespace and treat the placeholder token uniformly. Distinct - * placeholder PLACEMENTS remain distinct (only the placeholder glyph is - * unified, its position is preserved), so unrelated templates do not collapse. - * - * Pure string logic — used by the emerging_signals MCP tool. + * Sample + Drain-mine one time window and return the raw pattern groups + * (template + counts), without MCP response formatting. */ -export function normalizeTemplate(pattern: string): string { - return pattern - .replace(/<\*>/g, '\u0001') // stable placeholder marker - .replace(/\s+/g, ' ') - .trim() - .toLowerCase(); +export async function mineWindowPatterns( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + options?: WindowOptions, +): Promise<{ error: McpErrorResult } | MinedWindow> { + const sample = await fetchWindowSample( + teamId, + sourceId, + startDate, + endDate, + options, + ); + if ('error' in sample) return sample; + return mineWindowSample(sample, options); } // ─── Event pattern mining ──────────────────────────────────────────────────── diff --git a/packages/common-utils/src/__tests__/drain-masking.test.ts b/packages/common-utils/src/__tests__/drain-masking.test.ts index f79b05fc9..f882dd8d7 100644 --- a/packages/common-utils/src/__tests__/drain-masking.test.ts +++ b/packages/common-utils/src/__tests__/drain-masking.test.ts @@ -11,7 +11,7 @@ type Row = { body: string }; const mask = (s: string) => LogMasker.fromConfig(new TemplateMinerConfig()).mask(s); -const mine = (bodies: string[]) => +const mine = (bodies: string[], miner?: TemplateMiner) => minePatterns( bodies.map(body => ({ body })), { @@ -21,6 +21,7 @@ const mine = (bodies: string[]) => trendBuckets: 1, getBody: r => r.body, getTimestamp: () => null, + miner, }, ).patterns; @@ -253,4 +254,72 @@ describe('patterns with the default masks', () => { expect(day1).toEqual(day2); expect(day1.join('\n')).not.toMatch(/2026|Nov|Tue|Wed|[IE]110/); }); + + it('puts a token that varies in only one window into one cluster with a shared miner', () => { + const constant = Array.from( + { length: 20 }, + () => 'connect failed host=db-1 retry later', + ); + const varying = Array.from( + { length: 20 }, + (_, i) => `connect failed host=db-${i} retry later`, + ); + // Separate trees: the token stays literal in one window and becomes <*> + // in the other, so the templates differ. + expect(mine(constant)[0].pattern).not.toBe(mine(varying)[0].pattern); + + const miner = new TemplateMiner(new TemplateMinerConfig()); + const baseline = mine(constant, miner); + const current = mine(varying, miner); + expect(miner.drain.clusterCount).toBe(1); + expect(current.map(p => p.id)).toEqual(baseline.map(p => p.id)); + }); + + it('still tells new and gone patterns apart with one shared miner', () => { + const miner = new TemplateMiner(new TemplateMinerConfig()); + const baseline = mine( + [...dayOfLogs(3, 'Tue'), 'cache warmed in 120 ms'], + miner, + ); + const current = mine( + [...dayOfLogs(4, 'Wed'), 'disk quota exceeded on /data volume'], + miner, + ); + const baseIds = new Set(baseline.map(p => p.id)); + const curIds = new Set(current.map(p => p.id)); + const onlyCur = current.filter(p => !baseIds.has(p.id)); + const onlyBase = baseline.filter(p => !curIds.has(p.id)); + expect(onlyCur.map(p => p.pattern)).toEqual([ + 'disk quota exceeded on /data volume', + ]); + expect(onlyBase.map(p => p.pattern)).toEqual(['cache warmed in 120 ms']); + }); +}); + +describe('row cluster ids and tokens', () => { + it('returns the cluster id of each row, in input order', () => { + const rows = ['a b c', 'x y z', 'a b d'].map(body => ({ body })); + const opts = { + totalCount: 3, + startDate: new Date('2026-11-03T00:00:00Z'), + endDate: new Date('2026-11-05T00:00:00Z'), + trendBuckets: 1, + getBody: (r: Row) => r.body, + getTimestamp: () => null, + }; + const res = minePatterns(rows, opts); + const idOf = (prefix: string) => + Number(res.patterns.find(p => p.pattern.startsWith(prefix))?.id); + expect(res.rowClusterIds).toEqual([idOf('a'), idOf('x'), idOf('a')]); + expect(minePatterns([], opts).rowClusterIds).toEqual([]); + }); + + it('tokenizes a message the way Drain sees it, masks applied', () => { + const miner = new TemplateMiner(new TemplateMinerConfig()); + expect(miner.tokenize('2026-11-04T06:12:12Z sync failed')).toEqual([ + 'T06:12:12Z', + 'sync', + 'failed', + ]); + }); }); diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts index 505cf81c1..295af23ec 100644 --- a/packages/common-utils/src/drain/mine-patterns.ts +++ b/packages/common-utils/src/drain/mine-patterns.ts @@ -79,6 +79,8 @@ export interface MinePatternResult> { * `endDate`. */ trendBucketStarts: number[]; + /** The Drain cluster id of each input row, in input order. */ + rowClusterIds: number[]; } export interface MinePatternOptions> { @@ -102,6 +104,11 @@ export interface MinePatternOptions> { * Falls back to startDate if the extractor returns null/undefined. */ getTimestamp: (row: TRow) => number | null | undefined; + /** + * Drain miner to add the rows to. Default: a new one. Pass the same miner + * to several calls to get cluster ids that are comparable across them. + */ + miner?: TemplateMiner; } // ─── Core mining function ──────────────────────────────────────────────────── @@ -126,6 +133,7 @@ export function minePatterns>( maxSamples = 5, getBody, getTimestamp, + miner = new TemplateMiner(new TemplateMinerConfig()), } = options; const granularity = @@ -144,13 +152,10 @@ export function minePatterns>( sampleMultiplier: 1, trendGranularity: granularity, trendBucketStarts: allBuckets, + rowClusterIds: [], }; } - // ── Set up Drain miner ── - const drainConfig = new TemplateMinerConfig(); - const miner = new TemplateMiner(drainConfig); - // ── Process each row through Drain ── const clustered: Array<{ clusterId: number; @@ -239,5 +244,6 @@ export function minePatterns>( sampleMultiplier, trendGranularity: granularity, trendBucketStarts: allBuckets, + rowClusterIds: clustered.map(c => c.clusterId), }; } diff --git a/packages/common-utils/src/drain/template-miner.ts b/packages/common-utils/src/drain/template-miner.ts index 76e731dd4..6023e37ae 100644 --- a/packages/common-utils/src/drain/template-miner.ts +++ b/packages/common-utils/src/drain/template-miner.ts @@ -58,6 +58,11 @@ export class TemplateMiner { }; } + /** The masked tokens Drain compares for this message. */ + tokenize(logMessage: string): string[] { + return this.drain.getContentAsTokens(this.masker.mask(logMessage)); + } + match( logMessage: string, fullSearchStrategy: string = 'never',