diff --git a/src/agents/deterministicTester.test.ts b/src/agents/deterministicTester.test.ts index af45ef67..13abd7c5 100644 --- a/src/agents/deterministicTester.test.ts +++ b/src/agents/deterministicTester.test.ts @@ -171,3 +171,24 @@ describe('deterministic verification trust inputs', () => { )).rejects.toThrow(/verify-security: typecheck could not run inside the attested companion/); }); }); + +describe('llmFallback (AGT-4679)', () => { + const verify = { enabled: true, blockOnNewFailures: true, maxCommands: 4 }; + + it('does not ask the LLM tester when it is disabled and no deterministic command exists', async () => { + root = await mkdtemp(join(tmpdir(), 'openswarm-verify-nollm-')); + const fallback = vi.fn(); + const result = await runTesterWithVerification({ projectPath: root, verify, llmFallback: false, fallback }); + expect(fallback).not.toHaveBeenCalled(); + expect(result).toMatchObject({ success: true, testsPassed: 0, testsFailed: 0, deterministic: false }); + expect(result.output).toMatch(/nothing was verified/); + }); + + it('keeps the LLM tester as the fallback by default', async () => { + root = await mkdtemp(join(tmpdir(), 'openswarm-verify-llm-')); + const fallback = vi.fn().mockResolvedValue({ success: true, testsPassed: 1, testsFailed: 0, output: 'LLM tester' }); + const result = await runTesterWithVerification({ projectPath: root, verify, fallback }); + expect(fallback).toHaveBeenCalledOnce(); + expect(result.output).toBe('LLM tester'); + }); +}); diff --git a/src/agents/deterministicTester.ts b/src/agents/deterministicTester.ts index 1a1f6dcc..5881403b 100644 --- a/src/agents/deterministicTester.ts +++ b/src/agents/deterministicTester.ts @@ -187,6 +187,14 @@ export async function runTesterWithVerification(options: { trustedInputFingerprint?: string; fallback: () => Promise; onInfra?: (error: unknown) => void; + /** + * Whether the LLM tester may stand in when the deterministic verification cannot give a + * verdict. A role the operator disabled must not run, so the pipeline passes `false` then + * (AGT-4679): an infrastructure failure is rethrown and the attempt ends as `infra_error`, + * and a repository with no deterministic command gets a result that says nothing was verified. + * Default true keeps the old behaviour for everyone else. + */ + llmFallback?: boolean; }): Promise { if (options.verify?.enabled) { try { @@ -202,7 +210,20 @@ export async function runTesterWithVerification(options: { if (error instanceof VerifySecurityError) throw error; if (!isInfraError(error)) throw error; options.onInfra?.(error); + if (options.llmFallback === false) throw error; } } + if (options.llmFallback === false) return unverifiedResult(); return await options.fallback(); } + +/** Nothing could be run and no LLM was asked: say so rather than pass silently. */ +function unverifiedResult(): TesterResult { + return { + success: true, + testsPassed: 0, + testsFailed: 0, + output: 'No deterministic verification command exists for this repository and the LLM tester is disabled; nothing was verified.', + deterministic: false, + }; +} diff --git a/src/agents/pairPipeline.ts b/src/agents/pairPipeline.ts index 691dde2e..c9f6a461 100644 --- a/src/agents/pairPipeline.ts +++ b/src/agents/pairPipeline.ts @@ -331,11 +331,13 @@ export class PairPipeline extends EventEmitter { verify: this.config.verify, trustedCommands: context.trustedVerifyCommands, trustedPackageJsonByDirectory: context.trustedVerifyPackageJsonByDirectory, trustedInputFingerprint: context.trustedVerifyInputFingerprint, + // Only an explicit `enabled: false` turns the LLM tester off (AGT-4679). + llmFallback: this.config.roles?.tester?.enabled !== false, onInfra: (error) => { // The fallback changes what "tester passed" means — an LLM opinion in // place of ruff/pytest — so it goes to the stage log (stdout, dashboard) // and not only to stderr, where 29 of them went unnoticed (AGT-4416). - const line = `Deterministic verify unavailable; falling back to LLM tester: ${error instanceof Error ? error.message : String(error)}`; + const line = `Deterministic verify unavailable; ${this.config.roles?.tester?.enabled === false ? 'LLM tester is disabled, no verdict' : 'falling back to LLM tester'}: ${error instanceof Error ? error.message : String(error)}`; safeConsole.warn(`[${context.taskPrefix}] ${line}`); this.emit('log', { line }); }, diff --git a/src/agents/pairPipeline.verify.test.ts b/src/agents/pairPipeline.verify.test.ts index 1801097b..e1a1fa07 100644 --- a/src/agents/pairPipeline.verify.test.ts +++ b/src/agents/pairPipeline.verify.test.ts @@ -78,6 +78,7 @@ async function runPipeline(options: { verbose?: boolean; verify?: { enabled: boolean; blockOnNewFailures: boolean; maxCommands: number }; securityAudit?: { enabled: boolean; maxThreads: number }; + roles?: { tester?: { enabled?: boolean } }; } = {}) { const { PairPipeline } = await import('./pairPipeline.js'); const pipeline = new PairPipeline({ @@ -299,6 +300,30 @@ describe('PairPipeline deterministic tester (INT-2662)', () => { expect(logs).toEqual(expect.arrayContaining([expect.stringMatching(/^Deterministic verify unavailable; falling back to LLM tester: verify-runner: /)])); }); + it('does not fall back to the LLM tester when that role is disabled (AGT-4679)', async () => { + discoverVerifyCommands.mockResolvedValue([verifyCommand]); + runVerify.mockResolvedValue([{ + command: verifyCommand, + baseStatus: 'skipped', + headStatus: 'infra', + newFailure: false, + rawOutputTail: 'timeout after 20ms', + durationMs: 20, + }]); + const { result, logs } = await runPipeline({ roles: { tester: { enabled: false } } }); + expect(runTester).not.toHaveBeenCalled(); + expect(result.success).toBe(false); + expect(result.finalStatus).toBe('infra_error'); + expect(logs).toEqual(expect.arrayContaining([expect.stringMatching(/LLM tester is disabled, no verdict/)])); + }); + + it('passes without the LLM tester when it is disabled and no command exists (AGT-4679)', async () => { + const { result } = await runPipeline({ roles: { tester: { enabled: false } } }); + expect(runTester).not.toHaveBeenCalled(); + expect(result.success).toBe(true); + expect(result.testerResult?.output).toMatch(/nothing was verified/); + }); + it('downgrades an unavailable baseline comparison to the LLM fallback', async () => { discoverVerifyCommands.mockResolvedValue([verifyCommand]); runVerify.mockResolvedValue([{