feat(verify): honour a disabled tester role so the LLM tester is no longer the fallback for a failed verification (AGT-4679)

src/agents/deterministicTester.test.ts

@@ -171,3 +171,24 @@ describe('deterministic verification trust inputs', () => {
     )).rejects.toThrow(/verify-security: typecheck could not run inside the attested companion/);
   });
 });
+
+describe('llmFallback (AGT-4679)', () => {
+  const verify = { enabled: true, blockOnNewFailures: true, maxCommands: 4 };
+
+  it('does not ask the LLM tester when it is disabled and no deterministic command exists', async () => {
+    root = await mkdtemp(join(tmpdir(), 'openswarm-verify-nollm-'));
+    const fallback = vi.fn();
+    const result = await runTesterWithVerification({ projectPath: root, verify, llmFallback: false, fallback });
+    expect(fallback).not.toHaveBeenCalled();
+    expect(result).toMatchObject({ success: true, testsPassed: 0, testsFailed: 0, deterministic: false });
+    expect(result.output).toMatch(/nothing was verified/);
+  });
+
+  it('keeps the LLM tester as the fallback by default', async () => {
+    root = await mkdtemp(join(tmpdir(), 'openswarm-verify-llm-'));
+    const fallback = vi.fn().mockResolvedValue({ success: true, testsPassed: 1, testsFailed: 0, output: 'LLM tester' });
+    const result = await runTesterWithVerification({ projectPath: root, verify, fallback });
+    expect(fallback).toHaveBeenCalledOnce();
+    expect(result.output).toBe('LLM tester');
+  });
+});

src/agents/deterministicTester.ts

@@ -187,6 +187,14 @@ export async function runTesterWithVerification(options: {
   trustedInputFingerprint?: string;
   fallback: () => Promise<TesterResult>;
   onInfra?: (error: unknown) => void;
+  /**
+   * Whether the LLM tester may stand in when the deterministic verification cannot give a
+   * verdict. A role the operator disabled must not run, so the pipeline passes `false` then
+   * (AGT-4679): an infrastructure failure is rethrown and the attempt ends as `infra_error`,
+   * and a repository with no deterministic command gets a result that says nothing was verified.
+   * Default true keeps the old behaviour for everyone else.
+   */
+  llmFallback?: boolean;
 }): Promise<TesterResult> {
   if (options.verify?.enabled) {
     try {
@@ -202,7 +210,20 @@ export async function runTesterWithVerification(options: {
       if (error instanceof VerifySecurityError) throw error;
       if (!isInfraError(error)) throw error;
       options.onInfra?.(error);
+      if (options.llmFallback === false) throw error;
     }
   }
+  if (options.llmFallback === false) return unverifiedResult();
   return await options.fallback();
 }
+
+/** Nothing could be run and no LLM was asked: say so rather than pass silently. */
+function unverifiedResult(): TesterResult {
+  return {
+    success: true,
+    testsPassed: 0,
+    testsFailed: 0,
+    output: 'No deterministic verification command exists for this repository and the LLM tester is disabled; nothing was verified.',
+    deterministic: false,
+  };
+}

src/agents/pairPipeline.ts

@@ -331,11 +331,13 @@ export class PairPipeline extends EventEmitter {
       verify: this.config.verify,
       trustedCommands: context.trustedVerifyCommands, trustedPackageJsonByDirectory: context.trustedVerifyPackageJsonByDirectory,
       trustedInputFingerprint: context.trustedVerifyInputFingerprint,
+      // Only an explicit `enabled: false` turns the LLM tester off (AGT-4679).
+      llmFallback: this.config.roles?.tester?.enabled !== false,
       onInfra: (error) => {
         // The fallback changes what "tester passed" means — an LLM opinion in
         // place of ruff/pytest — so it goes to the stage log (stdout, dashboard)
         // and not only to stderr, where 29 of them went unnoticed (AGT-4416).
-        const line = `Deterministic verify unavailable; falling back to LLM tester: ${error instanceof Error ? error.message : String(error)}`;
+        const line = `Deterministic verify unavailable; ${this.config.roles?.tester?.enabled === false ? 'LLM tester is disabled, no verdict' : 'falling back to LLM tester'}: ${error instanceof Error ? error.message : String(error)}`;
         safeConsole.warn(`[${context.taskPrefix}] ${line}`);
         this.emit('log', { line });
       },

src/agents/pairPipeline.verify.test.ts

@@ -78,6 +78,7 @@ async function runPipeline(options: {
   verbose?: boolean;
   verify?: { enabled: boolean; blockOnNewFailures: boolean; maxCommands: number };
   securityAudit?: { enabled: boolean; maxThreads: number };
+  roles?: { tester?: { enabled?: boolean } };
 } = {}) {
   const { PairPipeline } = await import('./pairPipeline.js');
   const pipeline = new PairPipeline({
@@ -299,6 +300,30 @@ describe('PairPipeline deterministic tester (INT-2662)', () => {
     expect(logs).toEqual(expect.arrayContaining([expect.stringMatching(/^Deterministic verify unavailable; falling back to LLM tester: verify-runner: /)]));
   });
 
+  it('does not fall back to the LLM tester when that role is disabled (AGT-4679)', async () => {
+    discoverVerifyCommands.mockResolvedValue([verifyCommand]);
+    runVerify.mockResolvedValue([{
+      command: verifyCommand,
+      baseStatus: 'skipped',
+      headStatus: 'infra',
+      newFailure: false,
+      rawOutputTail: 'timeout after 20ms',
+      durationMs: 20,
+    }]);
+    const { result, logs } = await runPipeline({ roles: { tester: { enabled: false } } });
+    expect(runTester).not.toHaveBeenCalled();
+    expect(result.success).toBe(false);
+    expect(result.finalStatus).toBe('infra_error');
+    expect(logs).toEqual(expect.arrayContaining([expect.stringMatching(/LLM tester is disabled, no verdict/)]));
+  });
+
+  it('passes without the LLM tester when it is disabled and no command exists (AGT-4679)', async () => {
+    const { result } = await runPipeline({ roles: { tester: { enabled: false } } });
+    expect(runTester).not.toHaveBeenCalled();
+    expect(result.success).toBe(true);
+    expect(result.testerResult?.output).toMatch(/nothing was verified/);
+  });
+
   it('downgrades an unavailable baseline comparison to the LLM fallback', async () => {
     discoverVerifyCommands.mockResolvedValue([verifyCommand]);
     runVerify.mockResolvedValue([{