Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .changeset/patterns-mask-dated-lines.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/api': patch
Comment thread
pulpdrew marked this conversation as resolved.
Resolved
'@hyperdx/cli': patch
---

fix: MCP `event_patterns`, `emerging_signals` and search denoise, and the CLI,
no longer split the same log line by its date. Before mining, the Drain miner
masks the date of a timestamp (weekday, day, month, year) to `<DATE>`, so dated
lines from different days, months or years get the same template instead of one
"new" and one "gone". Times and ids are not masked. Within one day lines group
as before, except that klog lines with different severity letters are no longer
merged. No mask changes a line's token count: the `event_patterns` drill-down
predicate matches `<DATE>` as one or more non-whitespace characters, also inside
a token (the new `masks` option of the pattern predicate builders, which only
the MCP drill-down sets), and search denoise masks bodies the same way before
matching. Masking instructions get an optional `perWord` flag (default off, as
in drain3). The web app's Event patterns view uses its own drain3 config and is
unchanged.
1 change: 1 addition & 0 deletions packages/api/src/mcp/tools/query/denoise.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ export async function denoiseSearchResults(
const isNoisy = buildNoisyBodyMatcher(
patterns.map(p => p.pattern),
noisyTemplates,
{ masked: true },
);
const filteredRows = rows.filter(row => {
const bodyValue = row[bodyColumnKey];
Expand Down
5 changes: 4 additions & 1 deletion packages/api/src/mcp/tools/query/patternDrilldown.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import { renderWhereExpressionStr } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { DATE_TOKEN } from '@hyperdx/common-utils/dist/drain';
import type {
ChartVariable,
UseTextIndex,
Expand All @@ -9,7 +10,9 @@ import { substituteVariables } from '@hyperdx/common-utils/dist/variables';
export const PATTERN_MATCH_SEMANTICS =
'whereSnippet matches values with the same whitespace-separated token count as the ' +
'template, the same literal tokens (case-sensitive) in the same positions, and exactly ' +
'one non-whitespace token at each <*>. Every sampled member of the cluster matches. ' +
`one non-whitespace token at each <*>, and one or more non-whitespace characters at each ${DATE_TOKEN} ` +
`(also inside a token, e.g. backup_${DATE_TOKEN}.tar). ` +
'Every sampled member of the cluster matches. ' +
'It can also match unsampled events whose shape fits, including ones Drain would have ' +
'put in a more specific cluster, so the drill-down is a superset of cluster membership.';

Expand Down
5 changes: 4 additions & 1 deletion packages/api/src/mcp/tools/query/runEventPatterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
} from '@hyperdx/common-utils/dist/core/utils';
import {
buildPatternPredicate,
DATE_MASK,
MAX_PREDICATE_TEMPLATE_LENGTH,
minePatterns,
} from '@hyperdx/common-utils/dist/drain';
Expand Down Expand Up @@ -470,7 +471,9 @@ export async function runEventPatterns(
const slicedPatterns = rawPatterns.slice(0, topN);

const patterns = slicedPatterns.map((p, i) => {
const predicate = buildPatternPredicate(bodyColumn, p.pattern);
const predicate = buildPatternPredicate(bodyColumn, p.pattern, {
masks: [DATE_MASK],
});
const whereSnippet =
predicate == null ? '' : andSql(drilldown?.where, predicate);

Expand Down
256 changes: 256 additions & 0 deletions packages/common-utils/src/__tests__/drain-masking.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
import {
DEFAULT_MASKING_INSTRUCTIONS,
TemplateMinerConfig,
} from '@/drain/config';
import { LogMasker } from '@/drain/masking';
import { minePatterns } from '@/drain/mine-patterns';
import { TemplateMiner } from '@/drain/template-miner';

type Row = { body: string };

const mask = (s: string) =>
LogMasker.fromConfig(new TemplateMinerConfig()).mask(s);

const mine = (bodies: string[]) =>
minePatterns<Row>(
bodies.map(body => ({ body })),
{
totalCount: bodies.length,
startDate: new Date('2026-11-03T00:00:00Z'),
endDate: new Date('2026-11-05T00:00:00Z'),
trendBuckets: 1,
getBody: r => r.body,
getTimestamp: () => null,
},
).patterns;

const pad = (n: number) => String(n).padStart(2, '0');

// One day of mixed logs: Java/log4j, access logs, syslog, klog, ISO-8601,
// HTTP, JavaScript and java.util.logging dates. As in real logs, every line
// has its own time and id.
// `day` is the day of November 2026; `weekday` its name.
const dayOfLogs = (day: number, weekday: string) => {
let n = 0;
const t = () => {
n++;
return `${pad(6 + Math.floor(n / 60))}:${pad(n % 60)}:${pad((n * 7) % 60)}`;
};
const hex = () =>
`${pad(n % 100)}bf92f3577b34da6a3ce929d0e0e47${pad(n % 100)}`;
return Array.from({ length: 12 }, (_, i) => [
`2026-11-${pad(day)} ${t()},345 ${hex()} INFO started`,
`2026-11-${pad(day)} ${t()},345 ${hex()} ERROR crashed`,
`2026-11-${pad(day)} ${t()} ok`,
`2026-11-${pad(day)} ${t()} queue_depth=${i}`,
`[2026-11-${pad(day)} ${t()},345][pool-1-thread-${i}] INFO request handled ok`,
`[${pad(day)}/Nov/2026:${t()} +0000] "GET /a" 200`,
`[${pad(day)}/Nov/2026:${t()} +0000] "POST /b" 500`,
`Nov ${String(day).padStart(2, ' ')} ${t()} host sshd[${100 + i}]: session opened`,
`I11${pad(day)} ${t()}.345678 1 main.go:42] sync done`,
`E11${pad(day)} ${t()}.345678 1 main.go:42] sync failed`,
`2026-11-${pad(day)}T${t()}Z request ok`,
`Date: ${weekday}, ${pad(day)} Nov 2026 ${t()} GMT`,
`${weekday} Nov ${pad(day)} 2026 ${t()} GMT+0000 (Coordinated Universal Time) tick`,
`Nov ${day}, 2026 ${t().replace(/^0/, '')} AM org.example.Main run`,
`Started job at ${i % 2 ? 'boot' : 'shutdown'}`,
]).flat();
};

// Groups of line indices that a miner put in the same cluster.
const clusterGroups = (config: TemplateMinerConfig, lines: string[]) => {
const miner = new TemplateMiner(config);
const ids = lines.map(line => miner.addLogMessage(line).clusterId);
const groups = new Map<number, number[]>();
ids.forEach((id, i) => groups.set(id, [...(groups.get(id) ?? []), i]));
return [...groups.values()].map(g => g.join(',')).sort();
};

describe('default masking', () => {
it.each([
['2026-11-04T06:12:12.345Z started', '<DATE>T06:12:12.345Z started'],
['2026-11-04 06:12:12,345 INFO x', '<DATE> 06:12:12,345 INFO x'],
[
'run_id=scheduled__2026-11-04T06:00:00+00:00',
'run_id=scheduled__<DATE>T06:00:00+00:00',
],
[
'Date: Wed, 04 Nov 2026 06:12:12 GMT',
'Date: <DATE> <DATE> <DATE> <DATE> 06:12:12 GMT',
],
['[04/Nov/2026:06:12:12 +0000] "GET /"', '[<DATE>:06:12:12 +0000] "GET /"'],
['04-Nov-2026 06:12:12.345 INFO main', '<DATE> 06:12:12.345 INFO main'],
['Nov 4 06:12:12 host sshd', '<DATE> <DATE> 06:12:12 host sshd'],
['I1104 06:12:12.345678 1 main.go', 'I<DATE> 06:12:12.345678 1 main.go'],
['I1104\t06:12:12.345678 1 main.go', 'I<DATE>\t06:12:12.345678 1 main.go'],
[
'26/11/04 16:09:18 WARN KafkaOffsetReader: retry',
'<DATE> 16:09:18 WARN KafkaOffsetReader: retry',
],
['11/04/2026 06:12:12 x', '<DATE> 06:12:12 x'],
['Nov 4 06:12:12 2026-11-04 x', '<DATE> <DATE> 06:12:12 <DATE> x'],
[
'Wed Nov 4 06:12:12 +0000 2026 x',
'<DATE> <DATE> <DATE> 06:12:12 +0000 <DATE> x',
],
['04.11.2026 06:12:12 x', '<DATE> 06:12:12 x'],
[
'04 Nov 2026 06:12:12,345 INFO job done',
'<DATE> <DATE> <DATE> 06:12:12,345 INFO job done',
],
['partition 2026-11-04 loaded', 'partition <DATE> loaded'],
['wrote backup_2026-11-04.tar', 'wrote backup_<DATE>.tar'],
[
'Wed Nov 4 06:12:12 2026 job done',
'<DATE> <DATE> <DATE> 06:12:12 <DATE> job done',
],
[
'at Wed Nov 04 06:12:12 UTC 2026 done',
'at <DATE> <DATE> <DATE> 06:12:12 UTC <DATE> done',
],
['2026/11/04 06:12:12 server started', '<DATE> 06:12:12 server started'],
[
'Wed Nov 04 2026 06:12:12 GMT+0000 (Coordinated Universal Time) tick',
'<DATE> <DATE> <DATE> <DATE> 06:12:12 GMT+0000 (Coordinated Universal Time) tick',
],
[
'Nov 4, 2026 6:12:12 PM org.example.Main run',
'<DATE> <DATE> <DATE> 6:12:12 PM org.example.Main run',
],
[
'[2026-11-04 06:12:12,345][pool-1-thread-3] INFO ok',
'[<DATE> 06:12:12,345][pool-1-thread-3] INFO ok',
],
])('masks %s', (input, expected) => {
expect(mask(input)).toBe(expected);
});

it.each([
'version v1.2.3 build 2026',
'flags 0x1f and 0xdeadbeef',
'commit abc1234 on main',
'took 12.345 ms at 10:30',
'Sun is shining on 4 hosts',
'Meeting on Mon at noon',
'sent 5 Mar items ok',
'F1234 not a klog date',
'06:12:12 GET 2048 bytes',
'trace 4bf92f3577b34da6a3ce929d0e0e4736 ok',
'req 123e4567-e89b-12d3-a456-426614174000 done',
'release 10.11.2026 notes',
'part 1234-56-78 x',
'ip 10.11.12.13 06:12:12 up',
])('leaves %s alone', input => {
expect(mask(input)).toBe(input);
});

it('masks a drain3-style instruction once per match unless perWord is set', () => {
const config = new TemplateMinerConfig();
config.maskingInstructions = [{ pattern: '\\d+ ms', maskWith: 'DUR' }];
expect(LogMasker.fromConfig(config).mask('took 12 ms')).toBe('took <DUR>');
config.maskingInstructions = [
{ pattern: '\\d+ ms', maskWith: 'DUR', perWord: true },
];
expect(LogMasker.fromConfig(config).mask('took 12 ms')).toBe(
'took <DUR> <DUR>',
);
});

it('never changes the word count of a line', () => {
for (const line of dayOfLogs(4, 'Wed')) {
expect(mask(line).trim().split(/\s+/)).toHaveLength(
line.trim().split(/\s+/).length,
);
}
});

it('stays linear on long runs of whitespace', () => {
// A lookbehind that ends in \s+ rescans the run at every position: 128 KiB
// took seconds that way.
const run = ' '.repeat(1 << 17);
const start = performance.now();
expect(mask(`Nov 4 06:12:12${run}2026`)).toBe(
`<DATE> <DATE> 06:12:12${run}<DATE>`,
);
expect(performance.now() - start).toBeLessThan(1000);
});

it('lets extractParameters read a per-word masked date back', () => {
const miner = new TemplateMiner();
const line = 'Date: Wed, 04 Nov 2026 06:12:12 GMT';
const { templateMined } = miner.addLogMessage(line);
expect(
miner.extractParameters(templateMined, line)?.map(p => p.value),
).toEqual(['Wed,', '04', 'Nov', '2026']);
});

it('is on by default', () => {
expect(new TemplateMinerConfig().maskingInstructions).toEqual(
DEFAULT_MASKING_INSTRUCTIONS,
);
});

it('is off for a drain3 JSON config that lists no masks, as in drain3', () => {
expect(TemplateMinerConfig.fromJSON({}).maskingInstructions).toEqual([]);
});

it('takes the masks a drain3 JSON config lists', () => {
expect(
TemplateMinerConfig.fromJSON({
masking_instructions: [{ regex_pattern: '\\d+', mask_with: 'NUM' }],
}).maskingInstructions,
).toEqual([{ pattern: '\\d+', maskWith: 'NUM' }]);
});
});

describe('patterns with the default masks', () => {
it('cluster one day of logs as without masks', () => {
const unmasked = new TemplateMinerConfig();
unmasked.maskingInstructions = [];
const isKlog = (line: string) => /^[IWEF]\d{4} /.test(line);
const lines = dayOfLogs(4, 'Wed').filter(line => !isKlog(line));
expect(clusterGroups(new TemplateMinerConfig(), lines)).toEqual(
clusterGroups(unmasked, lines),
);
});

it('keeps klog severities apart, which unmasked mining merges', () => {
// Unmasked, I1104 and E1104 both route as numbers in Drain's prefix tree;
// masked, I<DATE> and E<DATE> route by their severity letter.
const lines = dayOfLogs(4, 'Wed').filter(line => /^[IE]\d{4} /.test(line));
expect(
mine(lines)
.map(p => p.pattern)
.sort(),
).toEqual([
'E<DATE> <*> 1 main.go:42] sync failed',
'I<DATE> <*> 1 main.go:42] sync done',
]);
});

it('give the last day of a month and the first of the next the same templates', () => {
const templates = (day: string, weekday: string, rfcDay: string) =>
mine([
`${day} 06:12:11 host sshd[41]: session opened`,
`Date: ${weekday}, ${rfcDay} 2026 06:12:11 GMT`,
`${weekday} ${day} 06:12:11 2026 job done`,
`${rfcDay} 2026 06:12:11,345 INFO job done`,
])
.map(p => p.pattern)
.sort();
expect(templates('Oct 31', 'Sat', '31 Oct')).toEqual(
templates('Nov 1', 'Sun', '01 Nov'),
);
});

it('give the logs of two different days the same templates', () => {
const templates = (lines: string[]) =>
mine(lines)
.map(p => p.pattern)
.sort();
const day1 = templates(dayOfLogs(3, 'Tue'));
const day2 = templates(dayOfLogs(4, 'Wed'));
expect(day1).toEqual(day2);
expect(day1.join('\n')).not.toMatch(/2026|Nov|Tue|Wed|[IE]110/);
});
});
37 changes: 37 additions & 0 deletions packages/common-utils/src/__tests__/noisy-matcher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,43 @@ describe('buildNoisyBodyMatcher', () => {
expect(isNoisy('something unseen')).toBe(false);
});

it('removes dated and id-bearing members of noisy clusters mined with the default masks', () => {
const noisy = Array.from(
{ length: 40 },
(_, i) =>
`2026-11-0${3 + (i % 2)} 06:12:${String(i % 60).padStart(2, '0')},345 INFO heartbeat trace_id=4bf92f3577b34da6a3ce929d0e0e47${String(i).padStart(2, '0')} ok`,
);
const rare = '2026-11-04 06:13:00,001 ERROR payment failed for order 42';
const { patterns } = minePatterns(
[...noisy, rare].map(body => ({ body })),
{
totalCount: noisy.length + 1,
startDate: new Date(0),
endDate: new Date(1000),
trendBuckets: 0,
maxSamples: noisy.length + 1,
getBody: r => r.body,
getTimestamp: () => 0,
},
);
const heartbeat = patterns.find(p => p.pattern.includes('heartbeat'));
expect(heartbeat?.pattern).toContain('<DATE>');
const isNoisy = buildNoisyBodyMatcher(
patterns.map(p => p.pattern),
new Set([heartbeat!.pattern]),
{ masked: true },
);
for (const body of noisy) expect(isNoisy(body)).toBe(true);
expect(isNoisy(rare)).toBe(false);
});

it('leaves the body unmasked unless asked, for templates mined without masks', () => {
// drain3 keeps a date shared by every sampled line as a literal.
const template = '2026-11-04 <*> INFO heartbeat ok';
const isNoisy = buildNoisyBodyMatcher([template], new Set([template]));
expect(isNoisy('2026-11-04 06:12:01,345 INFO heartbeat ok')).toBe(true);
});

it('prefers the noisy template on a tie', () => {
const isNoisy = buildNoisyBodyMatcher(
['a <*> c', 'a b <*>'],
Expand Down
Loading
Loading