Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/cli-v1-agent-install/skill-template.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,10 @@ blue 'Submit' button"`. The agent does its own semantic match; a guessed label
- **1–2 assertions, at the end, on content existence** rather than UI copy /
formatting. (Quoting a literal you submitted earlier in the same plan, to verify
it round-trips, is fine.)
- **Keep each assertion single and decisive.** Avoid conditional or multi-branch
wording such as "either A or B", "if A then B", `unless`, or `whether` — it can
exhaust the frontend run budget before a verdict is emitted. `test lint` warns
on these patterns before the plan consumes run credits.
- **Assertion targets name the specific page region** (panel / tab / output area).
Otherwise the agent settles for "some element with that text is visible
anywhere," which passes for wrong reasons.
Expand Down
99 changes: 99 additions & 0 deletions src/commands/test.lint-warning.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import { mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

import { describe, expect, it } from 'vitest';

import { runLint } from './test.js';

describe('runLint assertion-complexity warnings', () => {
it('warns when a frontend assertion contains conditional or multi-branch wording', async () => {
const dir = mkdtempSync(join(tmpdir(), 'cli-lint-conditional-'));
const file = join(dir, 'plan.json');
writeFileSync(

Check failure on line 13 in src/commands/test.lint-warning.spec.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found writeFileSync from package "node:fs" with non literal argument at index 0
file,
JSON.stringify({
projectId: 'project_alice',
type: 'frontend',
name: 'Knowledge Web renders',
planSteps: [
{
type: 'assertion',
description: 'Verify either an interactive graph canvas or a clear empty-state message',
},
],
}),
'utf8',
);

const report = await runLint(
{ profile: 'default', output: 'json', debug: false, planFrom: file },
{ stdout: () => undefined },
);

expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] });
expect(report.warnings).toEqual([
expect.objectContaining({
field: 'planSteps[0].description',
reason: expect.stringContaining('single, decisive assertion'),
}),
]);
});

it('keeps text warnings on stderr and stdout machine-safe', async () => {
const dir = mkdtempSync(join(tmpdir(), 'cli-lint-warning-streams-'));
const file = join(dir, 'plan.json');
writeFileSync(

Check failure on line 46 in src/commands/test.lint-warning.spec.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found writeFileSync from package "node:fs" with non literal argument at index 0
file,
JSON.stringify({
projectId: 'project_alice',
type: 'frontend',
name: 'Knowledge Web renders',
planSteps: [
{
type: 'assertion',
description: 'Verify either an interactive graph canvas or a clear empty-state message',
},
],
}),
'utf8',
);
const stdout: string[] = [];
const stderr: string[] = [];

await runLint(
{ profile: 'default', output: 'text', debug: false, planFrom: file },
{ stdout: line => stdout.push(line), stderr: line => stderr.push(line) },
);

expect(stdout.join('\n')).toBe('1/1 valid, 0 problem(s)');
expect(stdout.join('\n')).not.toContain('[warning]');
expect(stderr.join('\n')).toContain('[warning]');
expect(stderr.at(-1)).toBe('1 warning(s)');
});

it('does not flag actions or single-outcome assertions', async () => {
const dir = mkdtempSync(join(tmpdir(), 'cli-lint-decisive-'));
const file = join(dir, 'plan.json');
writeFileSync(

Check failure on line 78 in src/commands/test.lint-warning.spec.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found writeFileSync from package "node:fs" with non literal argument at index 0
file,
JSON.stringify({
projectId: 'project_alice',
type: 'frontend',
name: 'Checkout works',
planSteps: [
{ type: 'action', description: 'Open the cart or return to the catalog' },
{ type: 'assertion', description: 'Verify the order total is visible' },
],
}),
'utf8',
);

const report = await runLint(
{ profile: 'default', output: 'json', debug: false, planFrom: file },
{ stdout: () => undefined },
);

expect(report).toEqual({ checked: 1, valid: 1, issues: [] });
});
});
61 changes: 60 additions & 1 deletion src/commands/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1107,7 +1107,7 @@
const absolute = resolveAbsolute(path);
let stat;
try {
stat = statSync(absolute);

Check failure on line 1110 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found statSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
Expand Down Expand Up @@ -1135,7 +1135,7 @@

function readCodeFile(path: string): string {
try {
return stripBom(readFileSync(resolveAbsolute(path), 'utf8'));

Check failure on line 1138 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found readFileSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
Expand Down Expand Up @@ -1358,7 +1358,7 @@

let stat;
try {
stat = statSync(absolute);

Check failure on line 1361 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found statSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
Expand Down Expand Up @@ -1388,7 +1388,7 @@

let raw;
try {
raw = stripBom(readFileSync(absolute, 'utf8'));

Check failure on line 1391 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found readFileSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const reason = err instanceof Error ? err.message : 'unknown error';
throw localValidationError('steps', `cannot read ${path}: ${reason}`);
Expand Down Expand Up @@ -1422,7 +1422,7 @@
requireArrayLength('planSteps', stepsRaw, { min: 1, max: MAX_PLAN_STEPS, itemNoun: 'step' });

for (let i = 0; i < stepsRaw.length; i += 1) {
const step = stepsRaw[i];

Check warning on line 1425 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Variable Assigned to Object Injection Sink
if (typeof step !== 'object' || step === null || Array.isArray(step)) {
throw localValidationError(`planSteps[${i}]`, 'must be an object', undefined, 'field');
}
Expand Down Expand Up @@ -1477,7 +1477,7 @@
if (!Array.isArray(stepsRaw)) return issues;

for (let i = 0; i < stepsRaw.length; i += 1) {
const step: unknown = stepsRaw[i];

Check warning on line 1480 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Variable Assigned to Object Injection Sink
if (typeof step !== 'object' || step === null || Array.isArray(step)) {
issues.push({ field: `planSteps[${i}]`, reason: 'must be an object' });
continue;
Expand Down Expand Up @@ -2658,7 +2658,7 @@

let stat;
try {
stat = statSync(absolute);

Check failure on line 2661 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found statSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
Expand Down Expand Up @@ -2688,7 +2688,7 @@

let raw;
try {
raw = stripBom(readFileSync(absolute, 'utf8'));

Check failure on line 2691 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found readFileSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const reason = err instanceof Error ? err.message : 'unknown error';
throw localValidationError('plan-from', `cannot read ${path}: ${reason}`);
Expand Down Expand Up @@ -2834,7 +2834,7 @@
itemNoun: 'step',
});
for (let i = 0; i < (obj.planSteps as unknown[]).length; i += 1) {
const step = (obj.planSteps as unknown[])[i];

Check warning on line 2837 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Variable Assigned to Object Injection Sink
if (typeof step !== 'object' || step === null || Array.isArray(step)) {
throw localValidationError(
`${prefix}planSteps[${i}]`,
Expand Down Expand Up @@ -2903,7 +2903,7 @@
);
if (Array.isArray(obj.planSteps)) {
for (let i = 0; i < obj.planSteps.length; i += 1) {
const step: unknown = obj.planSteps[i];

Check warning on line 2906 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Variable Assigned to Object Injection Sink
if (typeof step !== 'object' || step === null || Array.isArray(step)) {
issues.push({ field: `${prefix}planSteps[${i}]`, reason: 'must be an object' });
continue;
Expand Down Expand Up @@ -3832,7 +3832,7 @@

let stat;
try {
stat = statSync(absolute);

Check failure on line 3835 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Found statSync from package "node:fs" with non literal argument at index 0
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
Expand Down Expand Up @@ -3882,7 +3882,7 @@
for (let i = 0; i < lines.length; i += 1) {
let parsed: unknown;
try {
parsed = JSON.parse(lines[i]!);

Check warning on line 3885 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Generic Object Injection Sink
} catch (err) {
const reason = err instanceof Error ? err.message : 'unknown error';
throw localValidationError(`plans[${i}]`, `not valid JSON: ${reason}`);
Expand Down Expand Up @@ -3940,7 +3940,7 @@
const specs: CliPlanInput[] = [];
let skippedCount = 0;
for (let i = 0; i < entries.length; i += 1) {
const filename = entries[i]!;

Check warning on line 3943 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Generic Object Injection Sink
const filePath = join(absolute, filename);

let raw: string;
Expand Down Expand Up @@ -4742,6 +4742,42 @@
checked: number;
valid: number;
issues: CliLintIssue[];
/** Non-fatal plan-quality findings; omitted when there are none. */
warnings?: CliLintIssue[];
}

/**
* Conditional and multi-branch assertion wording makes the frontend agent
* explore several outcomes and can exhaust its step budget before it emits a
* verdict. This is deliberately a narrow heuristic: only assertion steps are
* considered, and the finding stays non-fatal because the plan is structurally
* valid and may still be intentional.
*/
const CONDITIONAL_ASSERTION_PATTERN =
/\b(?:either|or|otherwise|unless|whether)\b|\bif\b[^.!?]*\bthen\b/i;

function collectAssertionComplexityWarnings(
parsed: unknown,
prefix = '',
): Array<{ field: string; reason: string }> {
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return [];
const obj = parsed as Record<string, unknown>;
if (obj.type !== undefined && obj.type !== 'frontend') return [];
if (!Array.isArray(obj.planSteps)) return [];

const warnings: Array<{ field: string; reason: string }> = [];
obj.planSteps.forEach((step, index) => {
if (typeof step !== 'object' || step === null || Array.isArray(step)) return;
const candidate = step as Record<string, unknown>;
if (candidate.type !== 'assertion' || typeof candidate.description !== 'string') return;
if (!CONDITIONAL_ASSERTION_PATTERN.test(candidate.description)) return;
warnings.push({
field: `${prefix}planSteps[${index}].description`,
reason:
'uses conditional or multi-branch wording; choose one observable outcome and write a single, decisive assertion to avoid exhausting the frontend run budget',
});
});
return warnings;
}

/**
Expand Down Expand Up @@ -4795,6 +4831,7 @@
}

const issues: CliLintIssue[] = [];
const warnings: CliLintIssue[] = [];
let checked = 0;

const lintPlanFile = (file: string, path: string, specIndex?: number): void => {
Expand All @@ -4809,6 +4846,10 @@
for (const issue of collectPlanIssues(parsed, { specIndex })) {
issues.push({ file, ...issue });
}
const prefix = specIndex === undefined ? '' : `specs[${specIndex}].`;
for (const warning of collectAssertionComplexityWarnings(parsed, prefix)) {
warnings.push({ file, ...warning });
}
};

const lintStepsFile = (file: string, path: string): void => {
Expand All @@ -4823,6 +4864,9 @@
for (const issue of collectPlanStepsIssues(parsed)) {
issues.push({ file, ...issue });
}
for (const warning of collectAssertionComplexityWarnings(parsed)) {
warnings.push({ file, ...warning });
}
};

if (opts.planFrom !== undefined) {
Expand Down Expand Up @@ -4883,11 +4927,26 @@
for (const issue of collectPlanIssues(parsed, { specIndex: lineNo - 1 })) {
issues.push({ file, ...issue });
}
for (const warning of collectAssertionComplexityWarnings(parsed, `specs[${lineNo - 1}].`)) {
warnings.push({ file, ...warning });
}
}
}

const filesWithIssues = new Set(issues.map(issue => issue.file)).size;
const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues };
const report: CliLintReport = {
checked,
valid: checked - filesWithIssues,
issues,
...(warnings.length > 0 ? { warnings } : {}),
};
if (opts.output === 'text' && warnings.length > 0) {
const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
for (const warning of warnings) {
stderr(`[warning] ${warning.file}: ${warning.field}: ${warning.reason}`);
}
stderr(`${warnings.length} warning(s)`);
}
out.print(report, () =>
[
...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`),
Expand Down Expand Up @@ -9044,7 +9103,7 @@
// landed server-side for it to dedup against.
chunkResponses = [];
for (let idx = 0; idx < chunks.length; idx++) {
const chunk = chunks[idx]!;

Check warning on line 9106 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Generic Object Injection Sink
// Bound the per-chunk idempotency key to <=256 chars (mirrors the retry
// path). A long base key plus the `:chunkN` suffix could otherwise exceed
// the server cap and be rejected or truncated inconsistently.
Expand Down Expand Up @@ -9217,7 +9276,7 @@
// double-trigger a shared BE producer/teardown.
retryChunkResponses = [];
for (let idx = 0; idx < retryChunks.length; idx++) {
const chunk = retryChunks[idx]!;

Check warning on line 9279 in src/commands/test.ts

View workflow job for this annotation

GitHub Actions / ESLint Security (changed files)

Generic Object Injection Sink
// [P2] Bound the derived key to ≤256 chars. Caller-supplied keys may
// be up to 256 chars; appending the suffix could exceed the server
// limit and cause every retry to be rejected. Truncate the base key
Expand Down
Loading