From ae31311558af653b3b47353fc0ed013b6b79d5a8 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 20 Sep 2026 14:44:49 -0400 Subject: [PATCH 1/3] fix(coding-agent): preserve eval task services Stop agent control processes before reading durable shell handoffs. Preserve environment-owned services through verification and block Harbor verification when control shutdown is unverified. --- .../src/diffex_terminal_bench/adapter.py | 9 +- .../src/diffex_terminal_bench/benchmark.py | 4 +- .../src/diffex_terminal_bench/cleanup.cjs | 148 +++++++++++++----- .../src/diffex_terminal_bench/execution.py | 1 + .../src/diffex_terminal_bench/gate.py | 1 + .../src/diffex_terminal_bench/lifecycle.py | 61 ++++++++ .../src/diffex_terminal_bench/observe.mjs | 2 + .../src/diffex_terminal_bench/outcome.py | 21 ++- .../src/diffex_terminal_bench/run.cjs | 14 +- .../src/diffex_terminal_bench/trajectory.py | 5 +- evals/terminal-bench/tests/cleanup.test.cjs | 50 ++++++ evals/terminal-bench/tests/test_execution.py | 5 +- evals/terminal-bench/tests/test_gate.py | 7 +- evals/terminal-bench/tests/test_outcome.py | 16 +- .../tests/test_parent_exhaustion.py | 5 +- evals/terminal-bench/tests/test_recovery.py | 6 +- evals/terminal-bench/tests/test_sampling.py | 2 +- 17 files changed, 293 insertions(+), 64 deletions(-) create mode 100644 evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py create mode 100644 evals/terminal-bench/tests/cleanup.test.cjs diff --git a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py index e0b99b4eb..1cb80f0af 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py @@ -16,7 +16,7 @@ from diffex_terminal_bench.config import PINS from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SUBAGENT_TOOLS -from diffex_terminal_bench.outcome import parse_outcome, require_completed +from diffex_terminal_bench.outcome import DiffexControlStopError, parse_outcome, require_completed from diffex_terminal_bench.trajectory import export_trajectory @@ -105,6 +105,7 @@ async def run( "pins": PINS, "provider": provider, "model": model, "thinking": options.thinking, "tools": tools, "subagents": options.subagents, "project_resources": options.project_resources, + "background_process_lifetime": options.background_process_lifetime, "evolution": {"enabled": False}, "state_policy": "fresh-per-run", "credential_env_names": sorted(credentials), "instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(), @@ -179,6 +180,10 @@ async def run( if outcome["status"] == "completed": outcome.update(status="infrastructure_failure", reason="trajectory_export_failed") (self.logs_dir / run_dir.name / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + # Raise inside finally so even a pending Harbor deadline cannot + # convert unsafe cleanup into a verifier-eligible agent timeout. + if outcome.get("agent_execution_stopped") is not True: + raise DiffexControlStopError("Agent execution stop is unverified; verification is blocked") if execution["interruption"] == "cancelled": raise asyncio.CancelledError require_completed(outcome) @@ -206,6 +211,8 @@ async def _finish_run(self, environment: BaseEnvironment, run_dir: PurePosixPath outcome = parse_outcome(directory, execution["interruption"], execution["launcher_exit_code"]) if failures: outcome["recovery_errors"] = failures + if any(failure["phase"] == "cleanup" for failure in failures): + outcome.update(agent_execution_stopped=False, cleanup_confirmed=False) if not execution["interruption"]: outcome.update(status="infrastructure_failure", reason="recovery_failed") context.metadata = {**(context.metadata or {}), "diffex_outcome": outcome, "diffex_run_dir": str(run_dir)} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py b/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py index 96c46c32b..9e642bda8 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py @@ -72,7 +72,7 @@ def run_identity(config: BenchmarkConfig, digests: dict) -> dict: return {"adapter_commit": commit, "dataset": config.dataset.model_dump(), "task_digests": digests, "pins": PINS, "harbor_version": version("harbor"), "daytona_version": version("daytona"), "python_version": platform.python_version(), "uv_lock_sha256": sha256(PACKAGE / "uv.lock"), - "model": config.model, "policy": config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources"}), + "model": config.model, "policy": config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), "pilot_tasks": config.pilot_tasks, "backend": "daytona", "daytona_target": os.environ.get("DAYTONA_TARGET", "organization-default"), "daytona_api_url": os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api"), @@ -85,7 +85,7 @@ def trial_config(config: BenchmarkConfig, task_path: Path, directory: Path, labe agent = AgentConfig(name="oracle") if oracle else AgentConfig( import_path=DiffexAgent.import_path(), model_name=config.model, override_setup_timeout_sec=630, - kwargs=config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources"}), + kwargs=config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), # Variable references are serialized, never credential values. Harbor # also uses these names to scrub accidental credential output from logs. env={name: f"${{{name}}}" for name in PROVIDER_ENV[config.model.split("/", 1)[0]] if os.environ.get(name)}, diff --git a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs b/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs index fa33935a3..db2b6191d 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs @@ -19,60 +19,130 @@ function signal(pid, start, name) { try { process.kill(pid, name); } catch (error) { if (error.code !== "ESRCH") throw error; } } +function snapshot() { + const processes = new Map(); + for (const entry of readdirSync("/proc")) { + if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue; + try { + const stat = readFileSync(`/proc/${entry}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/); + if (fields[0] === "Z") continue; + const uid = Number(readFileSync(`/proc/${entry}/status`, "utf8").match(/^Uid:\s+(\d+)/m)?.[1]); + let env; + try { env = readFileSync(`/proc/${entry}/environ`, "utf8").split("\0"); } + catch (error) { + if (error.code !== "EACCES") throw error; + if (uid !== process.getuid()) continue; + } + processes.set(Number(entry), { pid: Number(entry), start: fields[19], group: Number(fields[2]), env }); + } catch (error) { + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } + } + return processes; +} + +function ownership(runDir) { + const directory = join(runDir, "ownership"); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter((name) => name.endsWith(".json")).map((name) => { + const record = JSON.parse(readFileSync(join(directory, name), "utf8")); + if (record.version !== 1 || name !== `${record.executionId}.json` || + !Number.isSafeInteger(record.processGroupId) || record.processGroupId <= 1 || + !Number.isSafeInteger(record.supervisorPid) || record.supervisorPid <= 1 || + !((record.owner === "environment" && record.state === "background") || + (record.owner === "session" && ["active", "terminated"].includes(record.state))) || + !Array.isArray(record.members) || (record.owner === "environment" && !record.members.length) || + record.members.some((member) => !Number.isSafeInteger(member.pid) || member.pid <= 1 || + member.processGroupId !== record.processGroupId || typeof member.startTime !== "string" || !member.startTime)) { + throw new Error("Invalid shell ownership record"); + } + return record; + }); +} + async function cleanup(runDir) { - // Prevent a delayed launcher from starting after transport cancellation. + // This barrier must precede discovery, including when the launcher has not started. writeFileSync(join(runDir, "stop-requested"), "", { mode: 0o600 }); - const marker = `DIFFEX_TB_RUN_DIR=${runDir}`; - let launcher; - if (existsSync(join(runDir, "launcher.json"))) launcher = JSON.parse(readFileSync(join(runDir, "launcher.json"), "utf8")); - if (launcher) signal(launcher.pid, launcher.start, "SIGTERM"); - const seen = new Map(); + const controls = new Map(); const deadline = Date.now() + 7000; + const boot = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const seen = new Map(); let remaining = []; + let retained = []; + let records = []; + let stopped = false; + let quietScans = 0; do { - const unreadable = new Map(); - for (const entry of readdirSync("/proc")) { - if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue; - try { - if (!identity(entry)) continue; - if (readFileSync(`/proc/${entry}/environ`, "utf8").split("\0").includes(marker)) { - const start = identity(entry); - if (start) seen.set(Number(entry), start); + // Re-read identities to close delayed spawn/publication races. Do not + // inspect handoffs until their writers have stopped publishing them. + for (const name of ["launcher", "cli"]) { + const path = join(runDir, `${name}.json`); + if (!existsSync(path)) continue; + const control = JSON.parse(readFileSync(path, "utf8")); + if (!Number.isSafeInteger(control.pid) || control.pid <= 1 || + typeof control.start !== "string" || !control.start) throw new Error("Invalid control identity"); + controls.set(name, control); + } + // Leave time for TERM-resistant active descendants after control shutdown. + const force = Date.now() >= deadline - 4000; + for (const control of controls.values()) signal(control.pid, control.start, force ? "SIGKILL" : "SIGTERM"); + remaining = [...controls.values()].filter((control) => identity(control.pid) === control.start).map((control) => control.pid); + if (!remaining.length) { + stopped = true; + records = ownership(runDir); + const processes = snapshot(); + const owned = new Map(); + for (const record of records) { + const members = [...processes.values()].filter((member) => member.group === record.processGroupId); + const known = members.some((member) => record.members.some((previous) => + previous.pid === member.pid && previous.startTime === `${boot}:${member.start}`) || + member.env?.includes(`DIFFEX_SHELL_EXECUTION_ID=${record.executionId}`)); + if (!known) continue; // A recycled group ID is not ownership evidence. + for (const member of members) { + if (owned.has(member.pid) && owned.get(member.pid).executionId !== record.executionId) { + throw new Error("Conflicting shell ownership records"); + } + if ([...controls.values()].some((control) => control.pid === member.pid && control.start === member.start)) { + throw new Error("Control process claimed by shell ownership"); + } + owned.set(member.pid, record); } - } catch (error) { - if (error.code === "EACCES") { - if (!identity(entry)) continue; - // Other sandbox users' processes are outside this trial launcher. - let status; - try { status = readFileSync(`/proc/${entry}/status`, "utf8"); } - catch (statusError) { if (statusError.code === "ENOENT") continue; throw statusError; } - if (Number(status.match(/^Uid:\s+(\d+)/m)?.[1]) !== process.getuid()) continue; - // An exiting process can revoke environ access before becoming - // a zombie. Retry until it exits or becomes readable; never - // claim cleanup while a same-user process remains uninspectable. - const start = identity(entry); - if (start) unreadable.set(Number(entry), start); - continue; + } + retained = []; + for (const member of processes.values()) { + const record = owned.get(member.pid); + if (record?.owner === "environment") { + retained.push({ pid: member.pid, startTime: `${boot}:${member.start}`, + processGroupId: member.group, executionId: record.executionId, owner: "environment" }); + } else if (record || member.env?.includes(`DIFFEX_TB_RUN_DIR=${runDir}`)) { + seen.set(member.pid, member.start); + } else if (!member.env) { + remaining.push(member.pid); // Never claim success with uninspectable same-user work. } - if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + for (const [pid, start] of seen) signal(pid, start, Date.now() >= deadline - 2000 ? "SIGKILL" : "SIGTERM"); + remaining.push(...[...seen].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid)); + // A parent can fork after /proc enumeration and before its signal. + // Scan again after all discovered active work has exited. + quietScans = remaining.length ? 0 : quietScans + 1; + if (quietScans >= 2) break; } - const force = Date.now() >= deadline - 2000; - for (const [pid, start] of seen) signal(pid, start, force ? "SIGKILL" : "SIGTERM"); - if (launcher && force) signal(launcher.pid, launcher.start, "SIGKILL"); - remaining = [...seen].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid); - remaining.push(...[...unreadable].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid)); - if (launcher && identity(launcher.pid) === launcher.start) remaining.push(launcher.pid); - if (remaining.length === 0) break; await delay(100); } while (Date.now() < deadline); - const report = { confirmed: remaining.length === 0, remaining_pids: remaining, tracked_processes: seen.size }; + const report = { + schema_version: 1, phase: "pre-verifier", stop_requested: true, + agent_execution_stopped: stopped && remaining.length === 0, + ownership_validated: stopped, remaining_agent_pids: [...new Set(remaining)], + control_processes: [...controls].map(([role, control]) => ({ role, ...control })), + retained_processes: retained, ownership_records: records, + }; writeFileSync(join(runDir, "cleanup.json"), JSON.stringify(report), { mode: 0o600 }); - if (!report.confirmed) process.exitCode = 1; + if (!report.agent_execution_stopped) process.exitCode = 1; } module.exports = { identity }; if (require.main === module) cleanup(process.argv[2]).catch((error) => { - writeFileSync(join(process.argv[2], "cleanup.json"), JSON.stringify({ confirmed: false, error: error.code ?? error.name, path: error.path }), { mode: 0o600 }); + writeFileSync(join(process.argv[2], "cleanup.json"), JSON.stringify({ schema_version: 1, phase: "pre-verifier", agent_execution_stopped: false, ownership_validated: false, error: error.code ?? error.name, path: error.path }), { mode: 0o600 }); process.exitCode = 1; }); diff --git a/evals/terminal-bench/src/diffex_terminal_bench/execution.py b/evals/terminal-bench/src/diffex_terminal_bench/execution.py index 10506fa86..057c5b77e 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/execution.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/execution.py @@ -25,6 +25,7 @@ class ExecutionOptions(InstalledAgentOptions): tools: str = "read,bash,edit,write,update_plan" subagents: Literal["disabled", "enabled"] = "disabled" project_resources: Literal["context-only"] = "context-only" + background_process_lifetime: Literal["environment"] = "environment" # Arbitrary native config and prompt templates could defeat trial isolation # or alter the benchmark instruction. Reject them in Harbor preflight too. config: None = None diff --git a/evals/terminal-bench/src/diffex_terminal_bench/gate.py b/evals/terminal-bench/src/diffex_terminal_bench/gate.py index fafad8f1d..2d4ea5365 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/gate.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/gate.py @@ -203,6 +203,7 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = tools = policy["tools"].split(",") + (list(SUBAGENT_TOOLS) if policy["subagents"] == "enabled" else []) for key, value in {"provider": provider, "model": model, "thinking": policy["thinking"], "subagents": policy["subagents"], "project_resources": policy["project_resources"], + "background_process_lifetime": policy["background_process_lifetime"], "pins": PINS, "tools": tools, "evolution": {"enabled": False}, "state_policy": "fresh-per-run"}.items(): if requested.get(key) != value: raise ValueError(f"Requested configuration differs from gate identity: {key}") diff --git a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py new file mode 100644 index 000000000..03c1e0943 --- /dev/null +++ b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py @@ -0,0 +1,61 @@ +"""Validate pre-verifier control shutdown separately from environment teardown.""" + +import json +from pathlib import Path + + +def read_cleanup(directory: Path) -> dict: + report = json.loads((directory / "cleanup.json").read_text()) + if (not isinstance(report, dict) or report.get("schema_version") != 1 + or report.get("phase") != "pre-verifier" or report.get("stop_requested") is not True + or report.get("agent_execution_stopped") is not True + or report.get("ownership_validated") is not True + or report.get("remaining_agent_pids") != [] or not (directory / "stop-requested").is_file()): + raise ValueError("Agent execution stop is unverified") + controls = report.get("control_processes") + records = report.get("ownership_records") + retained = report.get("retained_processes") + if not all(isinstance(value, list) for value in (controls, records, retained)): + raise ValueError("Missing ownership or control evidence") + expected = [] + for role in ("launcher", "cli"): + path = directory / f"{role}.json" + if path.exists(): + control = json.loads(path.read_text()) + if (not isinstance(control, dict) or type(control.get("pid")) is not int or control["pid"] <= 1 + or not isinstance(control.get("start"), str) or not control["start"]): + raise ValueError("Invalid control identity") + expected.append({"role": role, **control}) + if controls != expected: + raise ValueError("Control identities differ from shutdown evidence") + saved = [json.loads(path.read_text()) for path in sorted((directory / "ownership").glob("*.json"))] + if sorted(records, key=lambda record: record["executionId"]) != sorted(saved, key=lambda record: record["executionId"]): + raise ValueError("Ownership records differ from shutdown evidence") + by_id = {} + for record in records: + if (record.get("version") != 1 or not isinstance(record.get("executionId"), str) + or not record["executionId"] or record["executionId"] in by_id + or type(record.get("processGroupId")) is not int or record["processGroupId"] <= 1 + or type(record.get("supervisorPid")) is not int or record["supervisorPid"] <= 1 + or (record.get("owner"), record.get("state")) not in ( + ("environment", "background"), ("session", "active"), ("session", "terminated")) + or not isinstance(record.get("members"), list) + or (record["owner"] == "environment" and not record["members"])): + raise ValueError("Invalid ownership record") + for member in record["members"]: + if (type(member.get("pid")) is not int or member["pid"] <= 1 + or member.get("processGroupId") != record["processGroupId"] + or not isinstance(member.get("startTime"), str) or not member["startTime"]): + raise ValueError("Invalid owned process identity") + by_id[record["executionId"]] = record + seen = set() + for member in retained: + record = by_id.get(member.get("executionId"), {}) + if (member.get("owner") != "environment" or record.get("owner") != "environment" + or member.get("processGroupId") != record.get("processGroupId") + or type(member.get("pid")) is not int or member["pid"] <= 1 or member["pid"] in seen + or not isinstance(member.get("startTime"), str) or not member["startTime"] + or any(control["pid"] == member["pid"] for control in controls)): + raise ValueError("Retained process lacks unambiguous environment ownership") + seen.add(member["pid"]) + return report diff --git a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs index 43a5e1f95..cb2257694 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs @@ -23,6 +23,8 @@ export default function observe(api) { provider: ctx.model?.provider, model: ctx.model?.id, thinking: api.getThinkingLevel(), tools: api.getActiveTools(), cwd: ctx.cwd, root_session: rootSession, + background_process_lifetime: process.argv.includes("--background-process-lifetime") + ? process.argv[process.argv.indexOf("--background-process-lifetime") + 1] : null, instruction_transport: "native-file", instruction_delivered: prompt === `\n${instruction}\n\n`, instruction_sha256: createHash("sha256").update(instruction).digest("hex"), diff --git a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py index c1ae60304..1c506f8f0 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py @@ -6,11 +6,17 @@ from harbor.agents.installed.base import AgentAuthenticationError, NonZeroAgentExitCodeError +from diffex_terminal_bench.lifecycle import read_cleanup + class DiffexExecutionError(NonZeroAgentExitCodeError): """An agent infrastructure failure, not a completed task with reward zero.""" +class DiffexControlStopError(RuntimeError): + """Block Harbor verification when agent execution may still be running.""" + + def parse_outcome(directory: Path, interruption: str | None = None, launcher_exit_code: int | None = None) -> dict: result = {"status": "incomplete", "reason": "missing_evidence", "stream_complete": False, @@ -21,12 +27,14 @@ def parse_outcome(directory: Path, interruption: str | None = None, if not isinstance(process, dict): raise ValueError("invalid_process") result["process"] = process - cleanup = json.loads((directory / "cleanup.json").read_text()) - if not isinstance(cleanup, dict): - raise ValueError("invalid_cleanup") - result["cleanup_confirmed"] = cleanup.get("confirmed") is True except (OSError, ValueError): process = {} + try: + cleanup = read_cleanup(directory) + result.update(cleanup_confirmed=True, agent_execution_stopped=True, + retained_processes=cleanup["retained_processes"]) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + result.update(cleanup_confirmed=False, agent_execution_stopped=False) # Parse incrementally so long runs do not require the whole stream in memory. active = False ended = False @@ -132,7 +140,8 @@ def parse_outcome(directory: Path, interruption: str | None = None, effective = json.loads((directory / "effective.json").read_text()) if not isinstance(requested, dict) or not isinstance(effective, dict): raise ValueError("invalid_configuration") - matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256")) + matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime")) + matches = matches and requested.get("background_process_lifetime") == "environment" matches = matches and effective.get("release_agents_loaded") is True and effective.get("instruction_delivered") is True matches = matches and sorted(effective["tools"]) == sorted(requested["tools"]) except (OSError, ValueError, KeyError, TypeError): @@ -143,6 +152,8 @@ def parse_outcome(directory: Path, interruption: str | None = None, def require_completed(outcome: dict) -> None: + if outcome.get("agent_execution_stopped") is not True: + raise DiffexControlStopError("Agent execution stop is unverified; verification is blocked") if outcome["status"] == "completed": return message = f"Diffex execution {outcome['status']}: {outcome['reason']}; see retained run artifacts" diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs index c51bfd5ec..4cfb71124 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs @@ -40,6 +40,8 @@ const args = [ "--thinking", process.env.DIFFEX_TB_THINKING, "--tools", config.tools.join(","), "--session", join(runDir, "sessions", "root.jsonl"), "--session-dir", join(runDir, "sessions"), + "--background-process-lifetime", config.background_process_lifetime, + "--background-process-ownership-dir", join(runDir, "ownership"), "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--extension", join(runDir, "observe.mjs"), "--print", `@${join(runDir, "instruction.txt")}`, ]; @@ -52,6 +54,8 @@ writeFileSync(join(runDir, "process.json"), JSON.stringify({ const child = spawn("/usr/local/bin/diffex", args, { cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr], detached: true, }); +const childStart = child.pid ? identity(child.pid) : null; +if (child.pid) writeFileSync(join(runDir, "cli.json"), JSON.stringify({ pid: child.pid, start: childStart }), { mode: 0o600 }); closeSync(stdout); closeSync(stderr); let spawnError = null; @@ -61,9 +65,15 @@ function stop() { if (interrupted) return; interrupted = true; if (!child.pid) return; - try { process.kill(-child.pid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; } + // Signal only the identified CLI. Group signals could reach retained task + // services; independent recovery handles active descendants after CLI exit. + if (identity(child.pid) === childStart) { + try { process.kill(child.pid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; } + } killTimer = setTimeout(() => { - try { process.kill(-child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + if (identity(child.pid) === childStart) { + try { process.kill(child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + } }, 5000); } for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, stop); diff --git a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py b/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py index 3ba9a1b29..875216d12 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py @@ -400,7 +400,10 @@ def aggregate(document): accounting = {"complete": not gaps and not missing, "gaps": sorted(set(gaps)), "missing_fields": sorted(missing), "observed_usage": observed, "observed_model_usage": dict(by_model), "totals": totals} - root_document["extra"].update(accounting=accounting, execution_status=outcome.get("status")) + root_document["extra"].update(accounting=accounting, execution_status=outcome.get("status"), + background_process_lifetime=requested.get("background_process_lifetime"), + agent_execution_stopped=outcome.get("agent_execution_stopped"), + retained_processes=outcome.get("retained_processes", [])) root_document["final_metrics"] = { **{f"total_{field}": value for field, value in totals.items()}, "total_steps": len(root.steps), "extra": {"scope": "root_and_all_descendants", "observed_usage": observed, "accounting_complete": accounting["complete"]}, diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs new file mode 100644 index 000000000..0a449c88c --- /dev/null +++ b/evals/terminal-bench/tests/cleanup.test.cjs @@ -0,0 +1,50 @@ +// Run in a Linux container: node --test /eval/tests/cleanup.test.cjs. +const assert = require("node:assert/strict"); +const { spawn } = require("node:child_process"); +const { once } = require("node:events"); +const { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } = require("node:fs"); +const { createServer, connect } = require("node:net"); +const { tmpdir } = require("node:os"); +const { join } = require("node:path"); +const { test } = require("node:test"); +const { identity } = require("../src/diffex_terminal_bench/cleanup.cjs"); + +if (process.argv[2] === "service") { + process.on("SIGTERM", () => {}); + const server = createServer((socket) => socket.end(String(process.pid))); + server.listen(0, "127.0.0.1", () => process.send(server.address().port)); +} else { + test("pre-verifier cleanup preserves a transferred marked service", { skip: process.platform !== "linux" }, async () => { + const runDir = mkdtempSync(join(tmpdir(), "diffex-cleanup-")); + mkdirSync(join(runDir, "ownership")); + const service = spawn(process.execPath, [__filename, "service"], { + detached: true, stdio: ["ignore", "ignore", "inherit", "ipc"], + env: { ...process.env, DIFFEX_TB_RUN_DIR: runDir, DIFFEX_SHELL_EXECUTION_ID: "service" }, + }); + try { + const [port] = await once(service, "message"); + const start = identity(service.pid); + const member = { pid: service.pid, processGroupId: service.pid, + startTime: `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` }; + writeFileSync(join(runDir, "ownership/service.json"), JSON.stringify({ + version: 1, executionId: "service", owner: "environment", state: "background", + processGroupId: service.pid, supervisorPid: process.pid, members: [member], + })); + const before = connect(port, "127.0.0.1"); + assert.equal(String((await once(before, "data"))[0]), String(service.pid)); + before.destroy(); + const cleanup = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/cleanup.cjs"), runDir], { stdio: "inherit" }); + assert.equal((await once(cleanup, "exit"))[0], 0); + assert.equal(identity(service.pid), start, "the original service must survive cleanup"); + const verifier = connect(port, "127.0.0.1"); + assert.equal(String((await once(verifier, "data"))[0]), String(service.pid)); + verifier.destroy(); + } finally { + if (service.exitCode === null && service.signalCode === null) { + service.kill("SIGKILL"); + await once(service, "exit"); + } + rmSync(runDir, { recursive: true, force: true }); + } + }); +} diff --git a/evals/terminal-bench/tests/test_execution.py b/evals/terminal-bench/tests/test_execution.py index 257a9fbe1..a03fe9dd1 100644 --- a/evals/terminal-bench/tests/test_execution.py +++ b/evals/terminal-bench/tests/test_execution.py @@ -26,7 +26,7 @@ def setUp(self): # and a Linux smoke using the real launcher and provider. async def finish_run(agent, environment, run_dir, context, execution): (agent.logs_dir / run_dir.name).mkdir(parents=True, exist_ok=True) - return {"status": "completed"} + return {"status": "completed", "agent_execution_stopped": True} finish = patch.object(DiffexAgent, "_finish_run", autospec=True, side_effect=finish_run) finish.start() @@ -86,6 +86,7 @@ async def upload(source, target): self.assertEqual(configs[0]["credential_env_names"], ["OPENAI_API_KEY"]) self.assertNotIn("spawn_agent", configs[0]["tools"]) self.assertEqual(configs[0]["evolution"], {"enabled": False}) + self.assertEqual(configs[0]["background_process_lifetime"], "environment") calls = [call.kwargs for call in environment.exec.call_args_list if call.kwargs.get("env")] self.assertEqual(len(calls), 2) for call in calls: @@ -147,7 +148,7 @@ async def test_missing_credentials_and_ambiguous_models_fail_before_exec(self): environment.exec.assert_not_called() def test_preflight_rejects_implicit_or_unsafe_policy(self): - for kwargs in ({"thinking": "automatic"}, {"tools": "bash,bash"}, {"tools": "spawn_agent"}, + for kwargs in ({"background_process_lifetime": "session"}, {"thinking": "automatic"}, {"tools": "bash,bash"}, {"tools": "spawn_agent"}, {"project_resources": "all"}, {"subagents": True}, {"unknown": "option"}, {"config": {"evolution": {"enabled": True}}}, {"prompt_template_path": "template.txt"}): with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): diff --git a/evals/terminal-bench/tests/test_gate.py b/evals/terminal-bench/tests/test_gate.py index f55516bdd..891adbff1 100644 --- a/evals/terminal-bench/tests/test_gate.py +++ b/evals/terminal-bench/tests/test_gate.py @@ -20,6 +20,7 @@ from diffex_terminal_bench.gate import inventory, require_previous, sha256, stage_tasks, validate_trial, write_json from diffex_terminal_bench.outcome import parse_outcome from diffex_terminal_bench.trajectory import export_trajectory +from test_outcome import write_cleanup from test_trajectory import USAGE, assistant, entry, interrupted_child_records @@ -32,10 +33,10 @@ def setUp(self): self.run = self.directory / "agent/run-fixture" self.run.mkdir(parents=True) self.identity = {"model": "openai/gpt-5", "policy": {"thinking": "high", "tools": "bash", - "subagents": "disabled", "project_resources": "context-only"}, + "subagents": "disabled", "project_resources": "context-only", "background_process_lifetime": "environment"}, "release_agents_sha256": "release-hash", "task_digests": {"task": "sha256:task"}} requested = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], - "subagents": "disabled", "project_resources": "context-only", "pins": PINS, + "subagents": "disabled", "project_resources": "context-only", "background_process_lifetime": "environment", "pins": PINS, "evolution": {"enabled": False}, "state_policy": "fresh-per-run"} (self.run / "instruction.txt").write_text("task") (self.run / "system-prompt.txt").write_text("system") @@ -44,7 +45,7 @@ def setUp(self): write_json(self.run / "effective.json", {**requested, "release_agents_loaded": True, "instruction_delivered": True, "release_agents_sha256": "release-hash", "system_prompt_sha256": sha256(self.run / "system-prompt.txt")}) write_json(self.run / "process.json", {"exit_code": 0}) - write_json(self.run / "cleanup.json", {"confirmed": True}) + write_cleanup(self.run) message = assistant("response") events = [{"type": "session"}, {"type": "agent_start"}, {"type": "message_start", "message": message}, {"type": "message_end", "message": message}, {"type": "agent_end", "messages": [message]}, diff --git a/evals/terminal-bench/tests/test_outcome.py b/evals/terminal-bench/tests/test_outcome.py index 65b949f76..17be30101 100644 --- a/evals/terminal-bench/tests/test_outcome.py +++ b/evals/terminal-bench/tests/test_outcome.py @@ -7,7 +7,7 @@ from harbor.agents.installed.base import AgentAuthenticationError -from diffex_terminal_bench.outcome import DiffexExecutionError, parse_outcome, require_completed +from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError, parse_outcome, require_completed def attempt(reason="stop", error="", retry=False): @@ -17,16 +17,26 @@ def attempt(reason="stop", error="", retry=False): {"type": "agent_end", "messages": [message], "willRetry": retry}] +def write_cleanup(directory): + (directory / "stop-requested").touch() + report = {"schema_version": 1, "phase": "pre-verifier", "stop_requested": True, + "agent_execution_stopped": True, "ownership_validated": True, + "remaining_agent_pids": [], "control_processes": [], "retained_processes": [], + "ownership_records": []} + (directory / "cleanup.json").write_text(json.dumps(report)) + return report + + def write_evidence(directory, events=None): directory.mkdir(parents=True, exist_ok=True) - config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture"} + config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture", "background_process_lifetime": "environment"} for name, content in { "requested.json": config, "effective.json": {**config, "release_agents_loaded": True, "instruction_delivered": True}, "process.json": {"exit_code": 0, "signal": None, "spawn_error": None}, - "cleanup.json": {"confirmed": True}, }.items(): (directory / name).write_text(json.dumps(content)) + write_cleanup(directory) (directory / "stdout.jsonl").write_text("".join(json.dumps(event) + "\n" for event in [{"type": "session"}, *(events if events is not None else attempt()), {"type": "agent_settled"}])) diff --git a/evals/terminal-bench/tests/test_parent_exhaustion.py b/evals/terminal-bench/tests/test_parent_exhaustion.py index 3c2d9ec28..45faa6d58 100644 --- a/evals/terminal-bench/tests/test_parent_exhaustion.py +++ b/evals/terminal-bench/tests/test_parent_exhaustion.py @@ -11,6 +11,7 @@ from diffex_terminal_bench.dataset import load_config from diffex_terminal_bench.gate import read_json, sample_tasks, write_json from diffex_terminal_bench.outcome import parse_outcome, require_completed +from test_outcome import write_cleanup class ParentExhaustionTests(unittest.IsolatedAsyncioTestCase): @@ -51,9 +52,9 @@ async def run(): (directory / "stdout.jsonl").write_bytes(stdout) (directory / "stderr.log").write_bytes(stderr) write_json(directory / "process.json", {"exit_code": process.returncode}) - write_json(directory / "cleanup.json", {"confirmed": True}) + write_cleanup(directory) requested = {"provider": "faux", "model": "fixture", "thinking": "off", - "instruction_sha256": "fixture", "tools": []} + "instruction_sha256": "fixture", "tools": [], "background_process_lifetime": "environment"} write_json(directory / "requested.json", requested) write_json(directory / "effective.json", {**requested, "release_agents_loaded": True, "instruction_delivered": True}) diff --git a/evals/terminal-bench/tests/test_recovery.py b/evals/terminal-bench/tests/test_recovery.py index be7cab1c1..38d638f46 100644 --- a/evals/terminal-bench/tests/test_recovery.py +++ b/evals/terminal-bench/tests/test_recovery.py @@ -12,7 +12,7 @@ from harbor.models.agent.context import AgentContext from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.outcome import DiffexExecutionError +from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError from test_outcome import write_evidence from test_trajectory import assistant, entry @@ -135,14 +135,14 @@ async def execute(**kwargs): return ExecResult(return_code=1 if "cleanup.cjs" in kwargs["command"] else 0, stdout="", stderr="") self.environment.exec.side_effect = execute - with self.assertRaises(DiffexExecutionError): + with self.assertRaises(DiffexControlStopError): await self.agent.run("Task", self.environment, self.context) self.environment.download_dir.assert_awaited_once() self.assertEqual(self.context.metadata["diffex_outcome"]["status"], "infrastructure_failure") async def test_log_collection_failure_is_recorded_locally(self): self.environment.download_dir.side_effect = ConnectionError("fixture") - with self.assertRaises(DiffexExecutionError): + with self.assertRaises(DiffexControlStopError): await self.agent.run("Task", self.environment, self.context) record = json.loads(next(self.logs.glob("run-*/outcome.json")).read_text()) self.assertEqual(record["recovery_errors"][0]["phase"], "log_collection") diff --git a/evals/terminal-bench/tests/test_sampling.py b/evals/terminal-bench/tests/test_sampling.py index a97479936..a68717235 100644 --- a/evals/terminal-bench/tests/test_sampling.py +++ b/evals/terminal-bench/tests/test_sampling.py @@ -37,7 +37,7 @@ async def create(config): self.assertEqual(len(gate["identity"]["sample"]["tasks"]), self.sample_count) self.assertEqual(config.agent.model_name, self.config.model) self.assertEqual(config.agent.kwargs, self.config.policy.model_dump( - include={"thinking", "tools", "subagents", "project_resources"})) + include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"})) self.assertEqual(config.timeout_multiplier, 1) self.assertTrue(config.environment.delete) self.started.append(config.task.path.name) From b698d4fbf04074238322c74b866ee46cf88d908d Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 20 Sep 2026 15:04:24 -0400 Subject: [PATCH 2/3] fix(coding-agent): harden eval cleanup evidence - Close launcher startup races and block verification when process ownership cannot be reconciled. - Exercise surviving services with real Linux CLI outcomes and the pinned gRPC verifier; reject releases lacking ownership support. --- .../terminal-bench/scripts/smoke_outcomes.py | 69 +++++-- .../src/diffex_terminal_bench/cleanup.cjs | 7 +- .../src/diffex_terminal_bench/gate.py | 7 +- .../src/diffex_terminal_bench/lifecycle.py | 21 ++- .../src/diffex_terminal_bench/run.cjs | 17 +- .../diffex_terminal_bench/verify-install.cjs | 5 + evals/terminal-bench/tests/cleanup.test.cjs | 173 +++++++++++++++--- evals/terminal-bench/tests/faux-outcomes.cjs | 91 +++++++-- .../tests/fixtures/Dockerfile.source | 11 ++ .../tests/fixtures/grpc_server.py | 37 ++++ .../fixtures/kill-before-cli-publication.cjs | 12 ++ .../tests/fixtures/kv-store.proto | 10 + evals/terminal-bench/tests/test_gate.py | 15 +- .../terminal-bench/tests/test_installation.py | 28 +++ evals/terminal-bench/tests/test_outcome.py | 47 ++++- evals/terminal-bench/tests/test_recovery.py | 26 ++- 16 files changed, 508 insertions(+), 68 deletions(-) create mode 100644 evals/terminal-bench/tests/fixtures/Dockerfile.source create mode 100644 evals/terminal-bench/tests/fixtures/grpc_server.py create mode 100644 evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs create mode 100644 evals/terminal-bench/tests/fixtures/kv-store.proto diff --git a/evals/terminal-bench/scripts/smoke_outcomes.py b/evals/terminal-bench/scripts/smoke_outcomes.py index 5eadef6d3..345ea1ee4 100644 --- a/evals/terminal-bench/scripts/smoke_outcomes.py +++ b/evals/terminal-bench/scripts/smoke_outcomes.py @@ -18,15 +18,18 @@ from diffex_terminal_bench.adapter import DiffexAgent from diffex_terminal_bench.config import PINS +from diffex_terminal_bench.dataset import load_config, prepare_dataset +from diffex_terminal_bench.gate import sha256 from diffex_terminal_bench.outcome import DiffexExecutionError -async def smoke(platform): +async def smoke(platform, source_image=None, grpc_fixture=False): config = tomllib.loads(Path("smoke.toml").read_text()) session_id = f"diffex-outcomes-{uuid.uuid4().hex[:12]}" trial = TrialPaths(Path("results", session_id).resolve()) trial.mkdir() - report = {"platform": platform, "image": config["images"][platform], "pins": PINS, + report = {"platform": platform, "image": (source_image or config["images"][platform]), "pins": PINS, + "qualification": "source-only" if source_image else "pinned-release", "passed": False, "cleanup_passed": False, "outcomes": []} print(f"Outcome evidence: {trial.trial_dir}", flush=True) with tempfile.TemporaryDirectory(prefix="diffex-outcomes-") as temp: @@ -35,7 +38,7 @@ async def smoke(platform): overlay.write_text(f"services:\n main:\n platform: {platform}\n") environment = DockerEnvironment( environment_dir=directory, environment_name="diffex-outcomes-smoke", session_id=session_id, - trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=config["images"][platform]), + trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=(source_image or config["images"][platform])), extra_docker_compose=[overlay], mounts=[{"type": "bind", "source": str(trial.agent_dir), "target": "/logs/agent"}], ) @@ -43,12 +46,22 @@ async def smoke(platform): await asyncio.wait_for(environment.start(force_build=False), config["environment_timeout_sec"]) agent = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", extra_env={"OPENAI_API_KEY": "fixture-secret-never-log"}) - await asyncio.wait_for(agent.setup(environment), PINS["setup_timeout_sec"] + 15) + if source_image: + source_wrapper = directory / "source-cli" + source_wrapper.write_text('#!/bin/sh\nexport TSX_TSCONFIG_PATH=/repo/tsconfig.json\nexec node --import /repo/node_modules/tsx/dist/loader.mjs /repo/packages/coding-agent/src/cli.ts "$@"\n') + await environment.upload_file(source_wrapper, "/usr/local/bin/diffex") + await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex") + else: + await asyncio.wait_for(agent.setup(environment), PINS["setup_timeout_sec"] + 15) await environment.upload_file(Path("tests/faux-outcomes.cjs"), "/tmp/faux-outcomes.cjs") + await environment.upload_file(Path("src/diffex_terminal_bench/cleanup.cjs"), "/tmp/faux-cleanup.cjs") wrapper = directory / "diffex" wrapper.write_text( '#!/bin/sh\n' '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs prepare\n' + 'if [ "$(cat /tmp/outcome-scenario)" != interrupted-output ]; then\n' + ' exec /usr/local/bin/diffex-release "$@"\n' + 'fi\n' '/usr/local/bin/diffex-release "$@"\n' 'status=$?\n' '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs interrupt-output\n' @@ -59,17 +72,30 @@ async def smoke(platform): await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex && mkdir -p /task/work && chmod 777 /task/work") environment.task_env_config.workdir = "/task/work" await agent.exec_as_root(environment, command="nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs >/tmp/faux-outcomes.log 2>&1 name.endsWith(".json")).map((name) => { const record = JSON.parse(readFileSync(join(directory, name), "utf8")); - if (record.version !== 1 || name !== `${record.executionId}.json` || + if (record.version !== 1 || typeof record.executionId !== "string" || !record.executionId || + name !== `${record.executionId}.json` || !Number.isSafeInteger(record.processGroupId) || record.processGroupId <= 1 || !Number.isSafeInteger(record.supervisorPid) || record.supervisorPid <= 1 || !((record.owner === "environment" && record.state === "background") || @@ -80,7 +81,7 @@ async function cleanup(runDir) { const path = join(runDir, `${name}.json`); if (!existsSync(path)) continue; const control = JSON.parse(readFileSync(path, "utf8")); - if (!Number.isSafeInteger(control.pid) || control.pid <= 1 || + if (Object.keys(control).sort().join(",") !== "pid,start" || !Number.isSafeInteger(control.pid) || control.pid <= 1 || typeof control.start !== "string" || !control.start) throw new Error("Invalid control identity"); controls.set(name, control); } @@ -132,7 +133,7 @@ async function cleanup(runDir) { } while (Date.now() < deadline); const report = { schema_version: 1, phase: "pre-verifier", stop_requested: true, - agent_execution_stopped: stopped && remaining.length === 0, + agent_execution_stopped: stopped && remaining.length === 0 && quietScans >= 2, ownership_validated: stopped, remaining_agent_pids: [...new Set(remaining)], control_processes: [...controls].map(([role, control]) => ({ role, ...control })), retained_processes: retained, ownership_records: records, diff --git a/evals/terminal-bench/src/diffex_terminal_bench/gate.py b/evals/terminal-bench/src/diffex_terminal_bench/gate.py index 2d4ea5365..c4342982b 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/gate.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/gate.py @@ -178,6 +178,7 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = agent = directory / "agent" installation = read_json(agent / "setup/installation.json") expected = {"verified": True, "harbor_version": PINS["harbor_version"], + "background_process_lifetime": "environment", "node_version": f"v{PINS['node_version']}", "diffex_version": PINS["diffex_version"], "npm_integrity": PINS["diffex_integrity"], "npm_registry": PINS["npm_registry"]} if any(installation.get(key) != value for key, value in expected.items()): @@ -195,8 +196,12 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = outcome = read_json(run / "outcome.json") if outcome.get("status") != "completed" or outcome.get("recovery_errors"): raise ValueError("Diffex did not complete cleanly") - if parse_outcome(run, launcher_exit_code=outcome.get("launcher_exit_code"))["status"] != "completed": + native_outcome = parse_outcome(run, launcher_exit_code=outcome.get("launcher_exit_code")) + if native_outcome["status"] != "completed": raise ValueError("Native execution evidence does not establish completion") + if any(outcome.get(key) != native_outcome.get(key) for key in ( + "agent_execution_stopped", "cleanup_confirmed", "retained_processes")): + raise ValueError("Outcome differs from process ownership evidence") requested, effective = read_json(run / "requested.json"), read_json(run / "effective.json") provider, model = identity["model"].split("/", 1) policy = identity["policy"] diff --git a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py index 03c1e0943..6441f1236 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py @@ -6,7 +6,7 @@ def read_cleanup(directory: Path) -> dict: report = json.loads((directory / "cleanup.json").read_text()) - if (not isinstance(report, dict) or report.get("schema_version") != 1 + if (not isinstance(report, dict) or type(report.get("schema_version")) is not int or report["schema_version"] != 1 or report.get("phase") != "pre-verifier" or report.get("stop_requested") is not True or report.get("agent_execution_stopped") is not True or report.get("ownership_validated") is not True @@ -22,18 +22,31 @@ def read_cleanup(directory: Path) -> dict: path = directory / f"{role}.json" if path.exists(): control = json.loads(path.read_text()) - if (not isinstance(control, dict) or type(control.get("pid")) is not int or control["pid"] <= 1 + if (not isinstance(control, dict) or set(control) != {"pid", "start"} + or type(control.get("pid")) is not int or control["pid"] <= 1 or not isinstance(control.get("start"), str) or not control["start"]): raise ValueError("Invalid control identity") expected.append({"role": role, **control}) if controls != expected: raise ValueError("Control identities differ from shutdown evidence") - saved = [json.loads(path.read_text()) for path in sorted((directory / "ownership").glob("*.json"))] + process_path = directory / "process.json" + if process_path.exists(): + process = json.loads(process_path.read_text()) + roles = {control["role"] for control in controls} + if "launcher" not in roles or (type(process.get("exit_code")) is int + and not process.get("spawn_error") and "cli" not in roles): + raise ValueError("Missing launched control identity") + saved = [] + for path in sorted((directory / "ownership").glob("*.json")): + record = json.loads(path.read_text()) + if path.name != f"{record['executionId']}.json": + raise ValueError("Ownership filename differs from execution identity") + saved.append(record) if sorted(records, key=lambda record: record["executionId"]) != sorted(saved, key=lambda record: record["executionId"]): raise ValueError("Ownership records differ from shutdown evidence") by_id = {} for record in records: - if (record.get("version") != 1 or not isinstance(record.get("executionId"), str) + if (type(record.get("version")) is not int or record["version"] != 1 or not isinstance(record.get("executionId"), str) or not record["executionId"] or record["executionId"] in by_id or type(record.get("processGroupId")) is not int or record["processGroupId"] <= 1 or type(record.get("supervisorPid")) is not int or record["supervisorPid"] <= 1 diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs index 4cfb71124..0421b896f 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs @@ -51,8 +51,13 @@ const stderr = openSync(join(runDir, "stderr.log"), "wx", 0o600); writeFileSync(join(runDir, "process.json"), JSON.stringify({ exit_code: null, signal: null, spawn_error: null, interrupted: false, }), { mode: 0o600 }); -const child = spawn("/usr/local/bin/diffex", args, { - cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr], detached: true, +// The child cannot execute the CLI before its identity is durable. If the +// launcher dies during publication, EOF on the private pipe closes the gate. +// exec preserves this PID and start time through the installed CLI wrapper. +const child = spawn("/bin/sh", ["-c", + 'read -r ready <&3 && [ "$ready" = start ] || exit 1; exec 3<&-; exec /usr/local/bin/diffex "$@"', + "diffex", ...args], { + cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr, "pipe"], detached: true, }); const childStart = child.pid ? identity(child.pid) : null; if (child.pid) writeFileSync(join(runDir, "cli.json"), JSON.stringify({ pid: child.pid, start: childStart }), { mode: 0o600 }); @@ -78,6 +83,7 @@ function stop() { } for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, stop); child.on("error", (error) => { spawnError = error.code ?? error.name; }); +child.stdio[3].on("error", (error) => { spawnError = error.code ?? error.name; }); child.on("close", (code, signal) => { clearTimeout(killTimer); writeFileSync(join(runDir, "process.json"), JSON.stringify({ @@ -86,4 +92,9 @@ child.on("close", (code, signal) => { process.exitCode = interrupted ? 1 : (code ?? 1); }); // Close the race with a stop request written during synchronous initialization. -if (existsSync(join(runDir, "stop-requested"))) stop(); +if (existsSync(join(runDir, "stop-requested"))) { + child.stdio[3].end(); + stop(); +} else { + child.stdio[3].end("start\n"); +} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs b/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs index 7918430c2..5fa6e9537 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs @@ -23,10 +23,15 @@ assert.equal(installed.resolved, tarball, "Installed registry mismatch"); if (process.argv[2] === "installed") process.exit(0); assert.equal(process.argv[2], "complete"); assert.equal(readFileSync(join(logs, "version.stdout.txt"), "utf8").trim(), env.DIFFEX_VERSION); +const help = readFileSync(join(logs, "help.stdout.txt"), "utf8"); +for (const flag of ["--background-process-lifetime", "--background-process-ownership-dir"]) { + assert.ok(help.includes(flag), `Pinned Diffex release lacks ${flag}; publish and pin the corrected release`); +} assert.ok(Number(env.SETUP_ELAPSED_SEC) < Number(env.SETUP_TIMEOUT_SEC), "Setup exceeded deadline"); writeFileSync(join(logs, "installation.json"), `${JSON.stringify({ schema_version: 1, verified: true, + background_process_lifetime: "environment", started_at: env.SETUP_STARTED_AT, elapsed_sec: Number(env.SETUP_ELAPSED_SEC), setup_timeout_sec: Number(env.SETUP_TIMEOUT_SEC), diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs index 0a449c88c..6d624a8aa 100644 --- a/evals/terminal-bench/tests/cleanup.test.cjs +++ b/evals/terminal-bench/tests/cleanup.test.cjs @@ -6,6 +6,7 @@ const { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } = require( const { createServer, connect } = require("node:net"); const { tmpdir } = require("node:os"); const { join } = require("node:path"); +const { setTimeout: delay } = require("node:timers/promises"); const { test } = require("node:test"); const { identity } = require("../src/diffex_terminal_bench/cleanup.cjs"); @@ -14,37 +15,157 @@ if (process.argv[2] === "service") { const server = createServer((socket) => socket.end(String(process.pid))); server.listen(0, "127.0.0.1", () => process.send(server.address().port)); } else { - test("pre-verifier cleanup preserves a transferred marked service", { skip: process.platform !== "linux" }, async () => { + async function fixture(t) { const runDir = mkdtempSync(join(tmpdir(), "diffex-cleanup-")); mkdirSync(join(runDir, "ownership")); - const service = spawn(process.execPath, [__filename, "service"], { - detached: true, stdio: ["ignore", "ignore", "inherit", "ipc"], - env: { ...process.env, DIFFEX_TB_RUN_DIR: runDir, DIFFEX_SHELL_EXECUTION_ID: "service" }, + const children = []; + t.after(async () => { + for (const child of children) { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await once(child, "exit"); + } + } + rmSync(runDir, { recursive: true, force: true }); }); + return { + runDir, + async service(executionId, marked = true) { + const child = spawn(process.execPath, [__filename, "service"], { + detached: true, stdio: ["ignore", "ignore", "inherit", "ipc"], + env: { ...process.env, ...(marked ? { DIFFEX_TB_RUN_DIR: runDir } : {}), + DIFFEX_SHELL_EXECUTION_ID: executionId }, + }); + children.push(child); + const [port] = await once(child, "message", { signal: t.signal }); + const start = identity(child.pid); + const member = { pid: child.pid, processGroupId: child.pid, + startTime: `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` }; + return { child, port, start, member, executionId }; + }, + publish(service, owner = "environment", overrides = {}) { + const record = { version: 1, executionId: service.executionId, owner, + state: owner === "environment" ? "background" : "active", processGroupId: service.child.pid, + supervisorPid: process.pid, members: [service.member], ...overrides }; + writeFileSync(join(runDir, `ownership/${service.executionId}.json`), JSON.stringify(record)); + return record; + }, + async cleanup(expectedCode = 0) { + const child = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/cleanup.cjs"), runDir], { stdio: "inherit" }); + children.push(child); + assert.equal((await once(child, "exit", { signal: t.signal }))[0], expectedCode); + return JSON.parse(readFileSync(join(runDir, "cleanup.json"), "utf8")); + }, + }; + } + + async function verify(service) { + assert.equal(identity(service.child.pid), service.start, "the original service must survive cleanup"); + const socket = connect(service.port, "127.0.0.1"); try { - const [port] = await once(service, "message"); - const start = identity(service.pid); - const member = { pid: service.pid, processGroupId: service.pid, - startTime: `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` }; - writeFileSync(join(runDir, "ownership/service.json"), JSON.stringify({ - version: 1, executionId: "service", owner: "environment", state: "background", - processGroupId: service.pid, supervisorPid: process.pid, members: [member], - })); - const before = connect(port, "127.0.0.1"); - assert.equal(String((await once(before, "data"))[0]), String(service.pid)); - before.destroy(); - const cleanup = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/cleanup.cjs"), runDir], { stdio: "inherit" }); - assert.equal((await once(cleanup, "exit"))[0], 0); - assert.equal(identity(service.pid), start, "the original service must survive cleanup"); - const verifier = connect(port, "127.0.0.1"); - assert.equal(String((await once(verifier, "data"))[0]), String(service.pid)); - verifier.destroy(); + assert.equal(String((await once(socket, "data"))[0]), String(service.child.pid)); } finally { - if (service.exitCode === null && service.signalCode === null) { - service.kill("SIGKILL"); - await once(service, "exit"); - } - rmSync(runDir, { recursive: true, force: true }); + socket.destroy(); } + } + + const options = { skip: process.platform !== "linux", timeout: 15000 }; + test("pre-verifier cleanup preserves a transferred marked service", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + await verify(service); + const report = await f.cleanup(); + assert.equal(report.agent_execution_stopped, true); + assert.deepEqual(report.remaining_agent_pids, []); + assert.equal(report.retained_processes[0].pid, service.child.pid); + await verify(service); + assert.deepEqual(await f.cleanup(), report, "cleanup is idempotent"); + }); + + test("forced CLI death preserves handoffs and kills TERM-resistant unfinished work", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + const active = await f.service("active"); + f.publish(active, "session"); + const cli = await f.service("control"); + writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); + cli.child.kill("SIGKILL"); + await once(cli.child, "exit"); + const report = await f.cleanup(); + assert.equal(identity(active.child.pid), null); + assert.equal(report.control_processes[0].pid, cli.child.pid); + await verify(service); + }); + + test("cleanup stops TERM-resistant control activity before allowing verification", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + const cli = await f.service("control"); + writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); + const report = await f.cleanup(); + assert.equal(report.agent_execution_stopped, true); + assert.equal(identity(cli.child.pid), null); + await verify(service); + }); + + test("PID reuse cannot signal an unrelated process or authorize a stale handoff", options, async (t) => { + const f = await fixture(t); + const unrelated = await f.service("unrelated", false); + writeFileSync(join(f.runDir, "launcher.json"), JSON.stringify({ pid: unrelated.child.pid, start: "recycled" })); + const active = await f.service("active"); + f.publish(active, "environment", { executionId: "stale", members: [{ ...active.member, startTime: "old-boot:0" }] }); + // The filename and token must refer to the stale execution, not the new one. + const stale = join(f.runDir, "ownership/active.json"); + writeFileSync(join(f.runDir, "ownership/stale.json"), readFileSync(stale)); + rmSync(stale); + const report = await f.cleanup(); + assert.deepEqual(report.retained_processes, []); + assert.equal(identity(active.child.pid), null); + await verify(unrelated); + }); + + test("invalid or contradictory handoff evidence fails closed without killing the service", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service, "environment", { state: "active" }); + assert.equal((await f.cleanup(1)).agent_execution_stopped, false); + await verify(service); + f.publish(service); + writeFileSync(join(f.runDir, "ownership/conflict.json"), JSON.stringify({ + ...f.publish(service), executionId: "conflict", owner: "session", state: "active", + })); + assert.equal((await f.cleanup(1)).agent_execution_stopped, false); + await verify(service); + }); + + test("stop barrier prevents a delayed real launcher from spawning the CLI", options, async (t) => { + const f = await fixture(t); + await f.cleanup(); + const launcher = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir]); + assert.equal((await once(launcher, "exit", { signal: t.signal }))[0], 1); + const report = await f.cleanup(); + assert.equal(report.control_processes[0].role, "launcher"); + assert.equal(report.agent_execution_stopped, true); + }); + + test("launcher death before PID publication cannot release the CLI gate", options, async (t) => { + const f = await fixture(t); + writeFileSync(join(f.runDir, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", + thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment" })); + writeFileSync(join(f.runDir, "release-agents.md"), "Fixture context"); + const launcher = spawn(process.execPath, ["--require", join(__dirname, "fixtures/kill-before-cli-publication.cjs"), + join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir], { + env: { ...process.env, DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "off" }, + }); + assert.equal((await once(launcher, "exit", { signal: t.signal }))[1], "SIGKILL"); + const child = JSON.parse(readFileSync(join(f.runDir, "gated-child.json"), "utf8")); + // EOF must stop the waiting child even before independent cleanup runs. + const deadline = Date.now() + 2000; + while (identity(child.pid) === child.start && Date.now() < deadline) await delay(10); + assert.notEqual(identity(child.pid), child.start); + assert.equal((await f.cleanup()).agent_execution_stopped, true); }); } diff --git a/evals/terminal-bench/tests/faux-outcomes.cjs b/evals/terminal-bench/tests/faux-outcomes.cjs index 1d94db012..80f7940ee 100644 --- a/evals/terminal-bench/tests/faux-outcomes.cjs +++ b/evals/terminal-bench/tests/faux-outcomes.cjs @@ -1,11 +1,43 @@ // Outcome smoke fixture. All model responses stay on container loopback. const { createServer } = require("node:http"); -const { spawn } = require("node:child_process"); +const { createServer: tcpServer, connect } = require("node:net"); +const assert = require("node:assert/strict"); +const { once } = require("node:events"); +const { setTimeout: delay } = require("node:timers/promises"); const { readFileSync, writeFileSync } = require("node:fs"); const { join, basename } = require("node:path"); +const { identity } = require("./faux-cleanup.cjs"); + const action = process.argv[2]; -if (action === "prepare") { +if (action === "health") { + (async () => { + for (let attempt = 0; attempt < 20; attempt++) { + try { + if ((await fetch("http://127.0.0.1:8766/health", { signal: AbortSignal.timeout(1000) })).ok) return; + } catch {} + await delay(100); + } + process.exitCode = 1; + })(); +} else if (action === "verify-service") { + (async () => { + const runDir = process.argv[3]; + const service = JSON.parse(readFileSync(join(runDir, "service.json"), "utf8")); + assert.equal(identity(service.pid), service.start); + for (const name of ["launcher", "cli"]) { + const control = JSON.parse(readFileSync(join(runDir, `${name}.json`), "utf8")); + assert.notEqual(identity(control.pid), control.start); + } + const socket = connect(service.port, "127.0.0.1"); + if (service.protocol === "grpc") await once(socket, "connect"); + else assert.equal(String((await once(socket, "data"))[0]), String(service.pid)); + socket.destroy(); + const provider = await (await fetch(`http://127.0.0.1:8766/state/${basename(runDir)}`)).json(); + assert.equal(provider.active, 0, "provider activity must stop before verification"); + console.log(JSON.stringify({ ...service, provider, verified: true })); + })().catch((error) => { console.error(error); process.exitCode = 1; }); +} else if (action === "prepare") { const state = process.env.DIFFEX_CODING_AGENT_DIR; const runDir = process.env.DIFFEX_TB_RUN_DIR; const scenario = readFileSync("/tmp/outcome-scenario", "utf8").trim(); @@ -17,8 +49,7 @@ if (action === "prepare") { }], } } })); process.exit(0); -} -if (action === "interrupt-output") { +} else if (action === "interrupt-output") { const runDir = process.env.DIFFEX_TB_RUN_DIR; if (readFileSync("/tmp/outcome-scenario", "utf8").trim() === "interrupted-output") { const path = join(runDir, "stdout.jsonl"); @@ -27,34 +58,66 @@ if (action === "interrupt-output") { writeFileSync(path, lines.filter((line) => JSON.parse(line).type !== "agent_end").join("\n") + '\n{"type":'); } process.exit(0); -} -if (action === "linger") { - // A detached tool descendant deliberately ignores TERM. +} else if (action === "linger") { + // Successful shell completion transfers this TERM-resistant service. process.on("SIGTERM", () => {}); - writeFileSync(join(process.env.DIFFEX_TB_RUN_DIR, "lingering-pid"), String(process.pid)); - setInterval(() => {}, 1000); -} else if (action === "spawn-linger") { - const child = spawn(process.execPath, [__filename, "linger"], { detached: true, stdio: "ignore" }); - child.unref(); + const runDir = process.env.DIFFEX_TB_RUN_DIR; + const server = tcpServer((socket) => socket.end(String(process.pid))); + server.listen(0, "127.0.0.1", () => writeFileSync(join(runDir, "service.json"), JSON.stringify({ + pid: process.pid, start: identity(process.pid), port: server.address().port, + }))); } else { const counts = new Map(); + const states = new Map(); createServer(async (request, response) => { if (request.url === "/health") { response.end("ok"); return; } + if (request.url.startsWith("/state/")) { + response.end(JSON.stringify(states.get(request.url.split("/")[2]))); + return; + } let text = ""; for await (const chunk of request) text += chunk; const body = JSON.parse(text); const scenario = request.url.split("/")[1]; const count = (counts.get(request.url) ?? 0) + 1; counts.set(request.url, count); + const run = request.url.split("/")[2]; + const state = states.get(run) ?? { requests: 0, active: 0 }; + states.set(run, state); + state.requests++; + state.active++; + response.on("close", () => state.active--); + if (["success", "service-failure", "timeout", "forced-death", "grpc"].includes(scenario) && count === 2) { + const runDir = join("/logs/agent", run); + const service = JSON.parse(readFileSync(join(runDir, "service.json"), "utf8")); + const socket = connect(service.port, "127.0.0.1"); + await once(socket, "connect"); + socket.destroy(); + writeFileSync(join(runDir, "service-before-exit.json"), JSON.stringify(service)); + if (scenario === "forced-death") { + const cli = JSON.parse(readFileSync(join(runDir, "cli.json"), "utf8")); + assert.equal(identity(cli.pid), cli.start); + process.kill(cli.pid, "SIGKILL"); + response.destroy(); + return; + } + } + if (scenario === "service-failure" && count > 1) { + response.writeHead(400, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ error: { message: "Controlled failure after handoff" } })); + return; + } if (scenario === "authentication" || scenario === "exhausted" || (scenario === "recovered" && count === 1)) { response.writeHead(scenario === "authentication" ? 401 : 503, { "Content-Type": "application/json" }); response.end(JSON.stringify({ error: { message: scenario === "authentication" ? "Controlled invalid API key" : "Controlled service overloaded" } })); return; } if (scenario === "timeout" && count > 1) return; - const tool = count === 1 && ["timeout", "success"].includes(scenario) - ? { name: "bash", arguments: JSON.stringify({ command: "/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs spawn-linger; printf retained-diagnostic >&2; printf tool-ok" }) } + let tool = count === 1 && ["timeout", "success", "service-failure", "forced-death"].includes(scenario) + ? { name: "bash", arguments: JSON.stringify({ command: 'nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs linger >/dev/null 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done; printf tool-ok' }) } : null; + if (scenario === "grpc" && count === 1) tool = { name: "bash", arguments: JSON.stringify({ command: + 'cd /app && /opt/verifier/bin/python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. kv-store.proto && nohup /opt/verifier/bin/python /app/server.py >/app/server.log 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done' }) }; response.writeHead(200, { "Content-Type": "text/event-stream" }); const send = (delta, finish_reason = null) => response.write(`data: ${JSON.stringify({ id: `fixture-${count}`, object: "chat.completion.chunk", model: body.model, diff --git a/evals/terminal-bench/tests/fixtures/Dockerfile.source b/evals/terminal-bench/tests/fixtures/Dockerfile.source new file mode 100644 index 000000000..1aff9db77 --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/Dockerfile.source @@ -0,0 +1,11 @@ +FROM node:22.23.2-bookworm-slim@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9 +WORKDIR /repo +COPY package.json package-lock.json tsconfig.json tsconfig.base.json ./ +COPY packages ./packages +RUN npm ci --ignore-scripts --no-audit --no-fund +RUN npm run hydrate:model-data +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && sed -i "s|http://deb.debian.org|https://deb.debian.org|g" /etc/apt/sources.list.d/debian.sources && apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv git procps && rm -rf /var/lib/apt/lists/* +RUN python3 -m venv /opt/verifier && /opt/verifier/bin/pip install pytest==8.4.2 grpcio==1.75.1 grpcio-tools==1.75.1 +RUN mkdir -p /opt/diffex-terminal-bench/node/bin /opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist && ln -s /usr/local/bin/node /opt/diffex-terminal-bench/node/bin/node +ENV TSX_TSCONFIG_PATH=/repo/tsconfig.json +WORKDIR /task diff --git a/evals/terminal-bench/tests/fixtures/grpc_server.py b/evals/terminal-bench/tests/fixtures/grpc_server.py new file mode 100644 index 000000000..cc8d422a9 --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/grpc_server.py @@ -0,0 +1,37 @@ +"""Known-correct service for the pinned verifier, with process identity evidence.""" + +from concurrent import futures +import json +import os +from pathlib import Path +import threading + +import grpc +import kv_store_pb2 +import kv_store_pb2_grpc + + +class Server(kv_store_pb2_grpc.KVStoreServicer): + def __init__(self): + self.values = {} + self.lock = threading.Lock() + + def GetVal(self, request, context): + with self.lock: + return kv_store_pb2.GetValResponse(val=self.values.get(request.key, 0)) + + def SetVal(self, request, context): + with self.lock: + self.values[request.key] = request.value + return kv_store_pb2.SetValResponse(val=request.value) + + +server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) +kv_store_pb2_grpc.add_KVStoreServicer_to_server(Server(), server) +assert server.add_insecure_port("127.0.0.1:5328") == 5328 +server.start() +stat = Path(f"/proc/{os.getpid()}/stat").read_text().rsplit(")", 1)[1].split() +Path(os.environ["DIFFEX_TB_RUN_DIR"], "service.json").write_text(json.dumps({ + "pid": os.getpid(), "start": stat[19], "port": 5328, "protocol": "grpc", +})) +server.wait_for_termination() diff --git a/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs b/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs new file mode 100644 index 000000000..b7228e47d --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs @@ -0,0 +1,12 @@ +// Terminate the real launcher at its spawn-to-publication boundary. +const fs = require("node:fs"); +const { basename, dirname, join } = require("node:path"); + +const write = fs.writeFileSync; +fs.writeFileSync = (path, ...args) => { + if (basename(path) === "cli.json") { + write(join(dirname(path), "gated-child.json"), ...args); + process.kill(process.pid, "SIGKILL"); + } + return write(path, ...args); +}; diff --git a/evals/terminal-bench/tests/fixtures/kv-store.proto b/evals/terminal-bench/tests/fixtures/kv-store.proto new file mode 100644 index 000000000..6e401bc3f --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/kv-store.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +service KVStore { + rpc GetVal (GetValRequest) returns (GetValResponse) {} + rpc SetVal (SetValRequest) returns (SetValResponse) {} +} +message GetValRequest { string key = 1; } +message GetValResponse { int32 val = 1; } +message SetValRequest { string key = 1; int32 value = 2; } +message SetValResponse { int32 val = 1; } diff --git a/evals/terminal-bench/tests/test_gate.py b/evals/terminal-bench/tests/test_gate.py index 891adbff1..d9802d394 100644 --- a/evals/terminal-bench/tests/test_gate.py +++ b/evals/terminal-bench/tests/test_gate.py @@ -20,7 +20,7 @@ from diffex_terminal_bench.gate import inventory, require_previous, sha256, stage_tasks, validate_trial, write_json from diffex_terminal_bench.outcome import parse_outcome from diffex_terminal_bench.trajectory import export_trajectory -from test_outcome import write_cleanup +from test_outcome import retain_service, write_cleanup from test_trajectory import USAGE, assistant, entry, interrupted_child_records @@ -62,6 +62,7 @@ def setUp(self): (self.run / name).touch() setup = self.directory / "agent/setup" write_json(setup / "installation.json", {"verified": True, "harbor_version": PINS["harbor_version"], + "background_process_lifetime": "environment", "node_version": f"v{PINS['node_version']}", "diffex_version": PINS["diffex_version"], "npm_integrity": PINS["diffex_integrity"], "npm_registry": PINS["npm_registry"], "elapsed_sec": 10}) for name in ("version", "help", "list-models"): @@ -287,6 +288,18 @@ def test_zero_reward_is_valid_benchmark_result(self): self.assertEqual(report["rewards"], {"reward": 0}) self.assertEqual(report["usage"]["prompt_tokens"], 19) + def test_service_handoff_qualifies_and_corrupt_ownership_cannot(self): + retain_service(self.run) + write_json(self.run / "outcome.json", parse_outcome(self.run, launcher_exit_code=0)) + self.reexport() + self.assertEqual(validate_trial(self.directory, "task", self.identity)["rewards"], {"reward": 0}) + self.mutate(self.run / "outcome.json", "retained_processes", []) + with self.assertRaisesRegex(ValueError, "ownership evidence"): + validate_trial(self.directory, "task", self.identity) + self.mutate(self.run / "ownership/service.json", "owner", "session") + with self.assertRaisesRegex(ValueError, "Native execution evidence"): + validate_trial(self.directory, "task", self.identity) + def test_missing_logs_and_forged_success_cannot_qualify(self): cases = [("agent/run-fixture/stderr.log", None, None), ("agent/run-fixture/process.json", "exit_code", 1), diff --git a/evals/terminal-bench/tests/test_installation.py b/evals/terminal-bench/tests/test_installation.py index 8dcde2a8a..bc965e9d4 100644 --- a/evals/terminal-bench/tests/test_installation.py +++ b/evals/terminal-bench/tests/test_installation.py @@ -53,6 +53,34 @@ async def test_execution_requires_explicit_model(self): class RegistryIntegrityTests(unittest.TestCase): + def test_release_without_ownership_flags_cannot_complete_installation(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + package = directory / "node_modules/@diffexai/diffex" + package.mkdir(parents=True) + tarball = f"{PINS['npm_registry']}@diffexai/diffex/-/diffex-{PINS['diffex_version']}.tgz" + installed = {"version": PINS["diffex_version"], "integrity": PINS["diffex_integrity"], "resolved": tarball} + (package / "package.json").write_text(json.dumps({"version": PINS["diffex_version"]})) + (directory / "package-lock.json").write_text(json.dumps({"packages": {"node_modules/@diffexai/diffex": installed}})) + (directory / "npm-metadata.json").write_text(json.dumps({"version": PINS["diffex_version"], + "dist": {"integrity": PINS["diffex_integrity"], "tarball": tarball}})) + (directory / "version.stdout.txt").write_text(PINS["diffex_version"]) + source = files("diffex_terminal_bench").joinpath("verify-install.cjs").read_text() + script = directory / "verify-install.cjs" + script.write_text(source.replace('"/opt/diffex-terminal-bench/cli"', json.dumps(temp))) + env = {"PATH": os.environ["PATH"], "SETUP_LOGS": temp, "NPM_REGISTRY": PINS["npm_registry"], + "DIFFEX_VERSION": PINS["diffex_version"], "DIFFEX_INTEGRITY": PINS["diffex_integrity"], + "SETUP_ELAPSED_SEC": "1", "SETUP_TIMEOUT_SEC": "600"} + for help_text, valid in (("--help", False), ("--background-process-lifetime", False), + ("--background-process-lifetime --background-process-ownership-dir", True)): + (directory / "help.stdout.txt").write_text(help_text) + result = subprocess.run(["node", str(script), "complete"], env=env, capture_output=True, text=True, timeout=10) + with self.subTest(help_text=help_text): + self.assertEqual(result.returncode == 0, valid, result.stderr) + self.assertEqual((directory / "installation.json").exists(), valid) + if not valid: + self.assertIn("publish and pin the corrected release", result.stderr) + def test_registry_metadata_must_match_all_pins(self): tarball = f"{PINS['npm_registry']}@diffexai/diffex/-/diffex-{PINS['diffex_version']}.tgz" cases = ( diff --git a/evals/terminal-bench/tests/test_outcome.py b/evals/terminal-bench/tests/test_outcome.py index 17be30101..be3c5667a 100644 --- a/evals/terminal-bench/tests/test_outcome.py +++ b/evals/terminal-bench/tests/test_outcome.py @@ -19,9 +19,12 @@ def attempt(reason="stop", error="", retry=False): def write_cleanup(directory): (directory / "stop-requested").touch() + controls = [{"role": "launcher", "pid": 10, "start": "100"}, {"role": "cli", "pid": 11, "start": "101"}] + for control in controls: + (directory / f"{control['role']}.json").write_text(json.dumps({key: value for key, value in control.items() if key != "role"})) report = {"schema_version": 1, "phase": "pre-verifier", "stop_requested": True, "agent_execution_stopped": True, "ownership_validated": True, - "remaining_agent_pids": [], "control_processes": [], "retained_processes": [], + "remaining_agent_pids": [], "control_processes": controls, "retained_processes": [], "ownership_records": []} (directory / "cleanup.json").write_text(json.dumps(report)) return report @@ -41,6 +44,18 @@ def write_evidence(directory, events=None): [{"type": "session"}, *(events if events is not None else attempt()), {"type": "agent_settled"}])) +def retain_service(directory): + report = write_cleanup(directory) + member = {"pid": 123, "processGroupId": 122, "startTime": "boot:456"} + record = {"version": 1, "executionId": "service", "owner": "environment", "state": "background", + "processGroupId": 122, "supervisorPid": 120, "members": [member]} + (directory / "ownership").mkdir(exist_ok=True) + (directory / "ownership/service.json").write_text(json.dumps(record)) + report.update(ownership_records=[record], retained_processes=[{**member, "executionId": "service", "owner": "environment"}]) + (directory / "cleanup.json").write_text(json.dumps(report)) + return report + + class OutcomeTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -120,6 +135,36 @@ def test_interruption_preserves_partial_evidence(self): self.assertFalse(outcome["stream_complete"]) self.assertTrue((self.directory / "stdout.jsonl").is_file()) + def test_retained_service_does_not_change_execution_outcome(self): + for interruption, code, status in ((None, 0, "completed"), (None, 1, "process_failure"), + ("cancelled", None, "cancelled"), ("timeout", None, "timeout")): + write_evidence(self.directory) + report = retain_service(self.directory) + outcome = parse_outcome(self.directory, interruption=interruption, launcher_exit_code=code) + self.assertEqual(outcome["status"], status) + self.assertTrue(outcome["agent_execution_stopped"]) + self.assertEqual(outcome["retained_processes"], report["retained_processes"]) + + def test_missing_or_contradictory_evidence_blocks_verification(self): + for key, value in (("agent_execution_stopped", False), ("ownership_validated", False), + ("remaining_agent_pids", [124]), ("ownership_records", []), + ("control_processes", []), + ("control_processes", [{"role": "cli", "pid": 123, "start": "456"}])): + write_evidence(self.directory) + report = retain_service(self.directory) + report[key] = value + (self.directory / "cleanup.json").write_text(json.dumps(report)) + for interruption in (None, "cancelled"): + with self.subTest(key=key, interruption=interruption): + outcome = parse_outcome(self.directory, interruption, launcher_exit_code=0) + self.assertFalse(outcome["agent_execution_stopped"]) + with self.assertRaises(DiffexControlStopError): + require_completed(outcome) + write_evidence(self.directory) + retain_service(self.directory) + (self.directory / "ownership/service.json").unlink() + self.assertEqual(parse_outcome(self.directory, launcher_exit_code=0)["status"], "cleanup_failure") + if __name__ == "__main__": unittest.main() diff --git a/evals/terminal-bench/tests/test_recovery.py b/evals/terminal-bench/tests/test_recovery.py index 38d638f46..0d8563dd5 100644 --- a/evals/terminal-bench/tests/test_recovery.py +++ b/evals/terminal-bench/tests/test_recovery.py @@ -5,11 +5,12 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import create_autospec +from unittest.mock import Mock, create_autospec from unittest.mock import patch from harbor.environments.base import BaseEnvironment, ExecResult from harbor.models.agent.context import AgentContext +from harbor.trial.single_step import SingleStepTrial from diffex_terminal_bench.adapter import DiffexAgent from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError @@ -147,6 +148,29 @@ async def test_log_collection_failure_is_recorded_locally(self): record = json.loads(next(self.logs.glob("run-*/outcome.json")).read_text()) self.assertEqual(record["recovery_errors"][0]["phase"], "log_collection") + async def test_deadline_with_failed_cleanup_blocks_real_harbor_verifier_path(self): + async def execute(**kwargs): + if "run.cjs" in kwargs["command"]: + await asyncio.Event().wait() + return ExecResult(return_code=1 if "cleanup.cjs" in kwargs["command"] else 0, stdout="", stderr="") + + self.environment.exec.side_effect = execute + trial = create_autospec(SingleStepTrial, instance=True) + trial._agent_timeout_sec = 1 + trial._load_trajectory = None + trial.result = self.context + trial.task = Mock() + + async def agent_phase(**kwargs): + await asyncio.wait_for(self.agent.run("Task", self.environment, self.context), 0.05) + + trial._run_agent_phase.side_effect = agent_phase + with self.assertRaises(DiffexControlStopError): + await SingleStepTrial._run_agent(trial) + trial._record_exception.assert_not_called() + trial._sync_agent_output.assert_awaited_once() + self.assertEqual(self.context.metadata["diffex_outcome"]["status"], "cancelled") + if __name__ == "__main__": unittest.main() From 067e020b1b7182dc32f1fd688f0494a707eb5eef Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 20 Sep 2026 15:06:36 -0400 Subject: [PATCH 3/3] docs(coding-agent): explain eval service lifetime Make environment ownership explicit in benchmark configurations. Document shutdown evidence, source regression results, and the required corrected release pin. --- evals/terminal-bench/README.md | 66 +++++++++++++++++++++++----- evals/terminal-bench/RELEASE_GATE.md | 12 ++++- evals/terminal-bench/benchmark.toml | 1 + evals/terminal-bench/execution.yaml | 1 + evals/terminal-bench/full.yaml | 1 + 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/evals/terminal-bench/README.md b/evals/terminal-bench/README.md index 6e16ca110..dc3824579 100644 --- a/evals/terminal-bench/README.md +++ b/evals/terminal-bench/README.md @@ -7,6 +7,9 @@ ATIF export and usage accounting run after evidence recovery, including interrup The [staged benchmark release gate](RELEASE_GATE.md) pins dataset inputs and verifies retained evidence before larger Daytona runs. Use the [offline job exporter](RELEASE_GATE.md#export-retained-trials-for-harbor-hub) to package existing stage results for Harbor Hub without rerunning evaluations. This integration is not yet ready for leaderboard runs. +The adapter now requires a release with `--background-process-lifetime` and `--background-process-ownership-dir` support. +The checked-in Diffex 0.3.0 pin predates these flags; installation deliberately fails its capability check until a corrected release and integrity are published and pinned. +Source-only lifecycle tests do not qualify the installed release or a live benchmark trial. ## Host setup @@ -121,7 +124,9 @@ Generated evidence is retained under `results/diffex-execution-*/`. Each execution smoke validates ATIF with Harbor's validator and reconciles nonzero input, output, cached-token, and cost totals against every root and child response. The outcome smoke exercises success, authentication failure, exhausted retry, recovered retry, interrupted JSON output, and a Harbor-style execution timeout as both root and a non-root user. It uses the pinned npm release with a loopback provider and checks nonzero terminal-error exits and rejection of truncated output even when the process exits zero. -Success and timeout cases create a detached tool descendant that ignores TERM, then verify that cleanup stops it before the next trial. +Success, provider-failure, forced-CLI-death, and timeout cases create a TERM-resistant background service and verify that its PID and start time are unchanged across adapter cleanup. +The verifier phase queries that same service and checks that the launcher, CLI, and provider requests have stopped. +The Linux cleanup regression independently creates a detached service with durable handoff evidence, and covers PID reuse, delayed launch, malformed records, and TERM-resistant active work. Evidence is retained under `results/diffex-outcomes-*/`, including partial stdout, stderr, native sessions, process status, cleanup, and outcomes. The outcome smoke also validates recoverable ATIF steps and checks that interrupted runs report incomplete accounting. The packaging check packs every public npm workspace into temporary tarballs with lifecycle scripts disabled, inspects their contents for benchmark paths, and records file counts and SHA-256 hashes in `results/production-contents.json`. @@ -129,6 +134,33 @@ It requires existing production build output, including `packages/coding-agent/d It does not build, publish, or regenerate release artifacts. Run `npm run check` from the repository root after changing code. +### Source lifecycle validation + +Until the corrected release is published, build the development-only image from the repository root: + +```bash +docker build --platform linux/arm64 -f evals/terminal-bench/tests/fixtures/Dockerfile.source -t diffex-adapter-tests:arm64 . +cd evals/terminal-bench +uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 +uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 --grpc-fixture +docker run --rm --init --user 65534:65534 -v "$PWD:/eval:ro" diffex-adapter-tests:arm64 node --test /eval/tests/cleanup.test.cjs +``` + +Repeat with `linux/amd64` and an `amd64` image tag for emulated coverage on an arm64 host. +Run the cleanup regression without `--user` for root coverage. +The development image installs dependencies without lifecycle scripts and runs the source CLI through `tsx`. +The gRPC fixture uses a known-correct implementation and all seven unmodified verifier tests extracted from the SHA-256-verified dataset archive. +It checks the original process identity before and after verifier RPCs and confirms container deletion afterward. +Every development smoke report is marked `qualification: source-only`. +After publication, update `pins.toml`, rerun both pinned-release Linux smokes, and start a fresh live benchmark trial before declaring integration qualification complete. + +On 2026-09-20, source-only validation passed 16 controlled outcome cases on each architecture, including root and UID 65534 execution. +Both architectures also passed all seven pinned gRPC verifier tests with unchanged service identity and confirmed container deletion. +The Linux cleanup suite passed seven deterministic cases, including forced death before CLI identity publication, and all 142 Python tests and `npm run check` passed. +The retained outcome reports are `results/diffex-outcomes-8dab466d6f5c/smoke.json` (arm64) and `results/diffex-outcomes-284ee6f0c224/smoke.json` (amd64). +The gRPC reports are `results/diffex-outcomes-8d83db9627fb/smoke.json` (arm64) and `results/diffex-outcomes-da1f6f5534a3/smoke.json` (amd64). +These local source checks leave pinned-release smokes and a fresh live benchmark trial pending. + ### Recorded validation The following historical smoke records used Diffex 0.2.2. @@ -270,7 +302,9 @@ Each run directory retains: - `effective.json`: resolved model, thinking, tools, working directory, session path, prompt hashes, and release-context verification. - `system-prompt.txt`: the effective system prompt, including release and task context. - `stdout.jsonl`, `stderr.log`, and `process.json`: native JSON events, diagnostics, and process status. -- `launcher.json` and `cleanup.json`: launcher process identity and verified cleanup status. +- `launcher.json` and `cli.json`: launcher and root CLI PID/start-time identities. +- `ownership/*.json`: durable shell ownership records published by the CLI. +- `stop-requested` and `cleanup.json`: the launch barrier, verified control shutdown, and explicitly retained environment-owned processes. - `launcher.stdout.log` and `launcher.stderr.log`: launcher diagnostics, including failures before CLI startup. - `outcome.json`: execution classification, stream completeness, final assistant stop reason, retry counts, interruption state, and recovery errors. - `accounting.json`: observed usage, nullable totals, per-model observations, and reasons accounting is incomplete. @@ -283,7 +317,8 @@ Provider usage fields in `AgentContext` contain reconciled sums of recorded usag ### Execution outcomes and interruption recovery -A completed execution requires zero CLI and launcher exit codes, no terminating signal or spawn error, matching effective configuration, and verified cleanup. +A completed execution requires zero CLI and launcher exit codes, no terminating signal or spawn error, matching effective configuration, and verified cessation of agent execution. +Environment-owned services may remain alive for shared-environment verification. The parser reads JSONL incrementally and requires matching assistant messages, a final `stop` response, closed message/run boundaries, resolved retries, and the native `agent_settled` event. Malformed or truncated lines, missing completion events, pending tool use, token-limit termination, and unresolved retries cannot count as completed runs. Earlier provider errors are recoverable when retry events and the subsequent terminal assistant state establish success. @@ -297,7 +332,8 @@ Tool errors alone do not fail execution: the assistant can inspect and recover f | Nonzero process, signal, or spawn error | `process_failure` | Raise `DiffexExecutionError` | | Missing, truncated, or unfinished native stream | `incomplete` | Raise `DiffexExecutionError` | | Effective configuration mismatch | `configuration_failure` | Raise `DiffexExecutionError` | -| Unverified cleanup or failed evidence collection | `cleanup_failure` / `infrastructure_failure` | Raise `DiffexExecutionError` | +| Unverified agent shutdown or missing shutdown evidence | `cleanup_failure` / `infrastructure_failure` | Raise `DiffexControlStopError`; block verification | +| Failed evidence collection with verified shutdown | `infrastructure_failure` | Raise `DiffexExecutionError` | | Harbor task deadline cancels `run()` | `cancelled` | Preserve cancellation; Harbor raises `AgentTimeoutError` | | Environment transport timeout or failure | `timeout` / `transport_failure` | Preserve the original exception | @@ -308,13 +344,23 @@ The pinned `0.3.0` release returns nonzero for terminal assistant errors and abo The adapter also parses native events to classify provider failures and reject incomplete output independently of process exit status. The launcher writes stdout and stderr directly to files and retains native sessions during execution. -On success, failure, or cancellation, an independent Harbor exec requests launcher shutdown, stops the CLI process group, and scans Linux `/proc` for same-user descendants carrying the trial's unique environment marker. -Process start times protect against signaling a reused PID. -Cleanup escalates from TERM to KILL after five seconds, waits up to two more seconds, and records whether any live tracked processes remain. +On success, failure, or cancellation, an independent Harbor exec writes the stop barrier and signals identified launcher and CLI processes individually. +A private pipe prevents CLI startup until the launcher has recorded its PID and start time; launcher death before publication closes the pipe without starting agent execution. +No launcher process-group signal can include a retained service. +After control processes exit, cleanup reads durable ownership records and scans Linux `/proc` for trial-marked processes and identified shell groups. +Linux boot ID, PID, start time, process group, and shell execution tokens distinguish transferred groups from reused process IDs. +Environment-owned groups survive; active or otherwise untransferred work receives TERM, escalating to KILL within a seven-second recovery budget. +Control escalation begins after three seconds, reserving time for active descendants; active-work escalation begins after five seconds. Already exited zombies cannot execute and are left for sandbox init to reap. -A stop-request file prevents delayed launch after transport cancellation. -Task commands must preserve the inherited trial environment marker for detached descendants to remain discoverable. -Harbor must still delete the sandbox after the trial for complete environment isolation. +The report records `agent_execution_stopped`, `ownership_validated`, `remaining_agent_pids`, control identities, retained service identities, and the consumed ownership records. +Outcome parsing and gate validation reconcile these records with retained files. +`cleanup_confirmed` in the outcome means pre-verifier agent shutdown, not an empty sandbox. +Missing, malformed, contradictory, or unverified shutdown evidence raises `DiffexControlStopError`, which is outside Harbor's verifier-eligible timeout and nonzero-exit exceptions. +This exception overrides a pending cancellation while retaining its interrupted outcome in the evidence. +Verified shutdown preserves Harbor's existing verification behavior for deadlines and nonzero agent exits without changing their failed classifications. +Task commands must preserve the inherited trial marker or remain in an identified shell group for unfinished descendants to remain discoverable. +Harbor stops the shared environment after verification, including verifier failure or timeout recovery; separate-environment verification keeps Harbor's artifact-based lifecycle. +The benchmark runner independently confirms sandbox deletion after every trial, including abandonment and transport failures. Recovery is shielded against repeated cancellation and runs before `run()` returns or re-raises the interruption. It allows up to 15 seconds for the cleanup transport and 15 seconds for mounted-log preparation or 30 seconds for remote log download. diff --git a/evals/terminal-bench/RELEASE_GATE.md b/evals/terminal-bench/RELEASE_GATE.md index c96ca2f7a..eae9a1f63 100644 --- a/evals/terminal-bench/RELEASE_GATE.md +++ b/evals/terminal-bench/RELEASE_GATE.md @@ -2,7 +2,8 @@ This gate qualifies a specific Diffex adapter, dataset, model, and harness configuration for a larger benchmark run. It does not publish a product release or submit a leaderboard entry. -The implementation supports all qualification stages; live Diffex qualification remains pending a model-provider credential. +The implementation supports all qualification stages; live Diffex qualification requires provider credentials and a published release with environment-owned process support. +The checked-in Diffex 0.3.0 pin lacks that support and deliberately fails installation capability validation until a corrected release and integrity are pinned. ## Configuration and prerequisites @@ -102,7 +103,8 @@ The Harbor authentication and upload commands below also apply to the exported s The adapter runs from the local uv project, and a newly started command reads the current Python source and `benchmark.toml`. An already running command keeps its loaded code and policy; let it finish or interrupt it once with Ctrl-C and wait for cleanup before restarting. -No Diffex npm release or product rebuild is needed for these runner and policy changes; new sandboxes install the existing pinned release with the updated tool allowlist. +The environment-owned process policy requires a corrected Diffex npm release and a reviewed pin update before new sandboxes can qualify. +Development source images exercise the lifecycle without qualifying the installed release. After updating the checkout, run the following in your Mac's zsh terminal with `DAYTONA_API_KEY` and `OPENAI_API_KEY` exported. `git status --short` must produce no output. @@ -245,6 +247,12 @@ Diffex installation evidence includes the three CLI checks and verified npm/Node The oracle does not install Diffex or call a model, so its Diffex installation, model usage, and ATIF checks are not applicable. The gate re-parses native process/stream evidence instead of trusting the outcome summary alone. +The execution policy includes `background_process_lifetime: environment` in the requested/effective configuration and benchmark identity. +Pre-verifier evidence must establish that all agent execution stopped, with every retained task service backed by a durable environment ownership record. +The gate reconciles control identities, ownership files, cleanup reports, outcome summaries, and trajectory metadata. +Retained services do not count as cleanup failures, and verifier reward cannot convert a deadline or provider failure into completed execution. +An unverified control stop raises `DiffexControlStopError` and prevents Harbor from entering verification. +Final cleanup remains Harbor's environment stop followed by independent sandbox-deletion confirmation. It regenerates ATIF into a temporary directory, validates it with Harbor, and compares it with retained trajectories and image attachments. Native totals, per-model totals, `accounting.json`, and `AgentContext` must reconcile. Unknown telemetry remains unknown and cannot satisfy complete accounting. diff --git a/evals/terminal-bench/benchmark.toml b/evals/terminal-bench/benchmark.toml index be514ea9e..b829ae23e 100644 --- a/evals/terminal-bench/benchmark.toml +++ b/evals/terminal-bench/benchmark.toml @@ -30,3 +30,4 @@ thinking = "max" tools = "read,bash,edit,write,update_plan" subagents = "enabled" project_resources = "context-only" +background_process_lifetime = "environment" diff --git a/evals/terminal-bench/execution.yaml b/evals/terminal-bench/execution.yaml index d3008b3ac..e564b896a 100644 --- a/evals/terminal-bench/execution.yaml +++ b/evals/terminal-bench/execution.yaml @@ -13,3 +13,4 @@ agents: tools: read,bash,edit,write,update_plan subagents: enabled project_resources: context-only + background_process_lifetime: environment diff --git a/evals/terminal-bench/full.yaml b/evals/terminal-bench/full.yaml index 5152d17b3..5a80919d8 100644 --- a/evals/terminal-bench/full.yaml +++ b/evals/terminal-bench/full.yaml @@ -14,6 +14,7 @@ agents: tools: read,bash,edit,write,update_plan subagents: enabled project_resources: context-only + background_process_lifetime: environment env: OPENAI_API_KEY: ${OPENAI_API_KEY} datasets: