Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/patterns-max-body-chars.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/api': patch
---

feat(mcp): `maxBodyChars` on `clickstack_event_patterns` and
`clickstack_emerging_signals` mines only the start of each body, so long stack
traces and dumps are cut in the query and their patterns can be drilled into
18 changes: 18 additions & 0 deletions packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,24 @@ describe('MCP Emerging Signals Tool', () => {
expect(checkoutEmerging).toBeUndefined();
});

it('mines only the first maxBodyChars characters of each body', async () => {
const result = await callTool(client, 'clickstack_emerging_signals', {
sourceId: logSource._id.toString(),
currentStartTime: new Date(now - 5 * 60_000).toISOString(),
currentEndTime: new Date(now).toISOString(),
baselineStartTime: new Date(now - 10 * 60_000).toISOString(),
baselineEndTime: new Date(now - 5 * 60_000).toISOString(),
sampleSize: 2000,
maxBodyChars: 12,
});

expect(result.isError).toBeFalsy();
const output = JSON.parse(getFirstText(result));
expect(output.summary.bodyColumn).toBe('leftUTF8(Body, 12)');
const emerging: EmergingPattern[] = output.emerging;
expect(emerging.map(p => p.sample)).toEqual(['shadow-eval ']);
});

it('reports nothing novel when both windows share the same patterns', async () => {
// Two non-overlapping halves of the baseline-only region (now-10min ..
// now-5min). Both contain only the steady-state checkout template and
Expand Down
30 changes: 30 additions & 0 deletions packages/api/src/mcp/__tests__/queryTool.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1301,6 +1301,36 @@ describe('MCP Query Tools', () => {
expect(p.drilldownUnavailable).toContain('longer than 4000');
});

it('mines the start of long bodies with maxBodyChars and drills by that prefix', async () => {
await bulkInsertLogs(
[101, 202, 303].map(id => ({
Body: `Error: job ${id} failed ${'at frame '.repeat(1000)}${id}`,
ServiceName: 'trace',
SeverityText: 'ERROR',
Timestamp: base,
})),
);
const output = await mine({
where: 'ServiceName:trace',
maxBodyChars: 27,
});
expect(output.summary.bodyColumn).toBe('leftUTF8(Body, 27)');
expect(output.patterns).toHaveLength(1);
const [p] = output.patterns;
expect(p.pattern).toBe('Error: job <*> failed at fr');
for (const sample of p.samples) expect(sample).toHaveLength(27);
const rows = await drill(output, p);
expect(rows).toHaveLength(3);
for (const r of rows) expect(r.Body.length).toBeGreaterThan(9000);

const result = await callTool(client, 'clickstack_event_patterns', {
sourceId: logSource._id.toString(),
...window,
maxBodyChars: 2.5,
});
expect(result.isError).toBe(true);
});

it('separates sampled counts from estimates', async () => {
const output = await mine({
where: 'ServiceName:svc-a',
Expand Down
12 changes: 12 additions & 0 deletions packages/api/src/mcp/tools/query/emergingSignals.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,17 @@ const emergingSignalsSchema = z.object({
'if omitted (Body for logs, SpanName for traces). ' +
'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']".',
),
maxBodyChars: z
.number()
.int()
.min(1)
.max(1_000_000)
.optional()
.describe(
'Mine only the first maxBodyChars characters of each value, e.g. 1000 for long ' +
'stack traces or request dumps. Long values are cut in the query, so less data ' +
'is read, and each sample is the cut value.',
),
currentStartTime: z
.string()
.describe(
Expand Down Expand Up @@ -204,6 +215,7 @@ export function registerEmergingSignals({
where: input.where,
whereLanguage: input.whereLanguage,
bodyExpression: input.bodyExpression,
maxBodyChars: input.maxBodyChars,
sampleSize: input.sampleSize,
trendBuckets: 0,
};
Expand Down
16 changes: 15 additions & 1 deletion packages/api/src/mcp/tools/query/eventPatterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,20 @@ const eventPatternsSchema = z.object({
'Examples: "Body", "SpanName", "LogAttributes[\'exception.type\']", ' +
"\"JSONExtractString(Body, 'error', 'message')\". Other function calls are rejected. " +
"Rows where the value is '' (missing key, invalid JSON) are mined as an empty " +
'pattern; add where "<expression> != \'\'" (whereLanguage:"sql") to exclude them.',
'pattern; add where "<expression> != \'\'" (whereLanguage:"sql") to exclude them. ' +
'To mine the start of long values, use maxBodyChars.',
),
maxBodyChars: z
.number()
.int()
.min(1)
.max(1_000_000)
.optional()
.describe(
'Mine only the first maxBodyChars characters of each value, e.g. 1000 for long ' +
'stack traces or request dumps. Long values are cut in the query, so less data ' +
'is read; samples are the cut values, and whereSnippet matches events by that ' +
'prefix (summary.bodyColumn shows the leftUTF8 expression used).',
),
topN: z
.number()
Expand Down Expand Up @@ -124,6 +137,7 @@ export function registerEventPatterns({
where: input.where,
whereLanguage: input.whereLanguage,
bodyExpression: input.bodyExpression,
maxBodyChars: input.maxBodyChars,
sampleSize: input.sampleSize,
topN: input.topN,
trendBuckets: input.trendBuckets,
Expand Down
10 changes: 10 additions & 0 deletions packages/api/src/mcp/tools/query/runEventPatterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ export async function mineWindowPatterns(
where?: string;
whereLanguage?: 'lucene' | 'sql';
bodyExpression?: string;
/** Mine only the first maxBodyChars characters of each body. */
maxBodyChars?: number;
sampleSize?: number;
trendBuckets?: number;
abortSignal?: AbortSignal;
Expand Down Expand Up @@ -152,6 +154,12 @@ export async function mineWindowPatterns(
),
};
}
// Truncate in the query, so long bodies are not transferred, and in the
// drill-down predicate, so it matches the prefix the template came from.
// The tool schemas bound maxBodyChars to an integer from 1 to 1,000,000.
if (options?.maxBodyChars != null) {
bodyColumn = `leftUTF8(${bodyColumn}, ${options.maxBodyChars})`;
}

const filters = getSourceTableFilters(source);
const tableFilterSql = andSql(
Expand Down Expand Up @@ -392,6 +400,7 @@ export async function runEventPatterns(
where?: string;
whereLanguage?: 'lucene' | 'sql';
bodyExpression?: string;
maxBodyChars?: number;
sampleSize?: number;
topN?: number;
trendBuckets?: number;
Expand All @@ -416,6 +425,7 @@ export async function runEventPatterns(
where: options?.where,
whereLanguage: options?.whereLanguage,
bodyExpression: options?.bodyExpression,
maxBodyChars: options?.maxBodyChars,
sampleSize: options?.sampleSize,
trendBuckets,
abortSignal: options?.abortSignal,
Expand Down
Loading