diff --git a/evals/terminal-bench/README.md b/evals/terminal-bench/README.md index 6e16ca110..dc3824579 100644 --- a/evals/terminal-bench/README.md +++ b/evals/terminal-bench/README.md @@ -7,6 +7,9 @@ ATIF export and usage accounting run after evidence recovery, including interrup The [staged benchmark release gate](RELEASE_GATE.md) pins dataset inputs and verifies retained evidence before larger Daytona runs. Use the [offline job exporter](RELEASE_GATE.md#export-retained-trials-for-harbor-hub) to package existing stage results for Harbor Hub without rerunning evaluations. This integration is not yet ready for leaderboard runs. +The adapter now requires a release with `--background-process-lifetime` and `--background-process-ownership-dir` support. +The checked-in Diffex 0.3.0 pin predates these flags; installation deliberately fails its capability check until a corrected release and integrity are published and pinned. +Source-only lifecycle tests do not qualify the installed release or a live benchmark trial. ## Host setup @@ -121,7 +124,9 @@ Generated evidence is retained under `results/diffex-execution-*/`. Each execution smoke validates ATIF with Harbor's validator and reconciles nonzero input, output, cached-token, and cost totals against every root and child response. The outcome smoke exercises success, authentication failure, exhausted retry, recovered retry, interrupted JSON output, and a Harbor-style execution timeout as both root and a non-root user. It uses the pinned npm release with a loopback provider and checks nonzero terminal-error exits and rejection of truncated output even when the process exits zero. -Success and timeout cases create a detached tool descendant that ignores TERM, then verify that cleanup stops it before the next trial. +Success, provider-failure, forced-CLI-death, and timeout cases create a TERM-resistant background service and verify that its PID and start time are unchanged across adapter cleanup. +The verifier phase queries that same service and checks that the launcher, CLI, and provider requests have stopped. +The Linux cleanup regression independently creates a detached service with durable handoff evidence, and covers PID reuse, delayed launch, malformed records, and TERM-resistant active work. Evidence is retained under `results/diffex-outcomes-*/`, including partial stdout, stderr, native sessions, process status, cleanup, and outcomes. The outcome smoke also validates recoverable ATIF steps and checks that interrupted runs report incomplete accounting. The packaging check packs every public npm workspace into temporary tarballs with lifecycle scripts disabled, inspects their contents for benchmark paths, and records file counts and SHA-256 hashes in `results/production-contents.json`. @@ -129,6 +134,33 @@ It requires existing production build output, including `packages/coding-agent/d It does not build, publish, or regenerate release artifacts. Run `npm run check` from the repository root after changing code. +### Source lifecycle validation + +Until the corrected release is published, build the development-only image from the repository root: + +```bash +docker build --platform linux/arm64 -f evals/terminal-bench/tests/fixtures/Dockerfile.source -t diffex-adapter-tests:arm64 . +cd evals/terminal-bench +uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 +uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 --grpc-fixture +docker run --rm --init --user 65534:65534 -v "$PWD:/eval:ro" diffex-adapter-tests:arm64 node --test /eval/tests/cleanup.test.cjs +``` + +Repeat with `linux/amd64` and an `amd64` image tag for emulated coverage on an arm64 host. +Run the cleanup regression without `--user` for root coverage. +The development image installs dependencies without lifecycle scripts and runs the source CLI through `tsx`. +The gRPC fixture uses a known-correct implementation and all seven unmodified verifier tests extracted from the SHA-256-verified dataset archive. +It checks the original process identity before and after verifier RPCs and confirms container deletion afterward. +Every development smoke report is marked `qualification: source-only`. +After publication, update `pins.toml`, rerun both pinned-release Linux smokes, and start a fresh live benchmark trial before declaring integration qualification complete. + +On 2026-09-20, source-only validation passed 16 controlled outcome cases on each architecture, including root and UID 65534 execution. +Both architectures also passed all seven pinned gRPC verifier tests with unchanged service identity and confirmed container deletion. +The Linux cleanup suite passed seven deterministic cases, including forced death before CLI identity publication, and all 142 Python tests and `npm run check` passed. +The retained outcome reports are `results/diffex-outcomes-8dab466d6f5c/smoke.json` (arm64) and `results/diffex-outcomes-284ee6f0c224/smoke.json` (amd64). +The gRPC reports are `results/diffex-outcomes-8d83db9627fb/smoke.json` (arm64) and `results/diffex-outcomes-da1f6f5534a3/smoke.json` (amd64). +These local source checks leave pinned-release smokes and a fresh live benchmark trial pending. + ### Recorded validation The following historical smoke records used Diffex 0.2.2. @@ -270,7 +302,9 @@ Each run directory retains: - `effective.json`: resolved model, thinking, tools, working directory, session path, prompt hashes, and release-context verification. - `system-prompt.txt`: the effective system prompt, including release and task context. - `stdout.jsonl`, `stderr.log`, and `process.json`: native JSON events, diagnostics, and process status. -- `launcher.json` and `cleanup.json`: launcher process identity and verified cleanup status. +- `launcher.json` and `cli.json`: launcher and root CLI PID/start-time identities. +- `ownership/*.json`: durable shell ownership records published by the CLI. +- `stop-requested` and `cleanup.json`: the launch barrier, verified control shutdown, and explicitly retained environment-owned processes. - `launcher.stdout.log` and `launcher.stderr.log`: launcher diagnostics, including failures before CLI startup. - `outcome.json`: execution classification, stream completeness, final assistant stop reason, retry counts, interruption state, and recovery errors. - `accounting.json`: observed usage, nullable totals, per-model observations, and reasons accounting is incomplete. @@ -283,7 +317,8 @@ Provider usage fields in `AgentContext` contain reconciled sums of recorded usag ### Execution outcomes and interruption recovery -A completed execution requires zero CLI and launcher exit codes, no terminating signal or spawn error, matching effective configuration, and verified cleanup. +A completed execution requires zero CLI and launcher exit codes, no terminating signal or spawn error, matching effective configuration, and verified cessation of agent execution. +Environment-owned services may remain alive for shared-environment verification. The parser reads JSONL incrementally and requires matching assistant messages, a final `stop` response, closed message/run boundaries, resolved retries, and the native `agent_settled` event. Malformed or truncated lines, missing completion events, pending tool use, token-limit termination, and unresolved retries cannot count as completed runs. Earlier provider errors are recoverable when retry events and the subsequent terminal assistant state establish success. @@ -297,7 +332,8 @@ Tool errors alone do not fail execution: the assistant can inspect and recover f | Nonzero process, signal, or spawn error | `process_failure` | Raise `DiffexExecutionError` | | Missing, truncated, or unfinished native stream | `incomplete` | Raise `DiffexExecutionError` | | Effective configuration mismatch | `configuration_failure` | Raise `DiffexExecutionError` | -| Unverified cleanup or failed evidence collection | `cleanup_failure` / `infrastructure_failure` | Raise `DiffexExecutionError` | +| Unverified agent shutdown or missing shutdown evidence | `cleanup_failure` / `infrastructure_failure` | Raise `DiffexControlStopError`; block verification | +| Failed evidence collection with verified shutdown | `infrastructure_failure` | Raise `DiffexExecutionError` | | Harbor task deadline cancels `run()` | `cancelled` | Preserve cancellation; Harbor raises `AgentTimeoutError` | | Environment transport timeout or failure | `timeout` / `transport_failure` | Preserve the original exception | @@ -308,13 +344,23 @@ The pinned `0.3.0` release returns nonzero for terminal assistant errors and abo The adapter also parses native events to classify provider failures and reject incomplete output independently of process exit status. The launcher writes stdout and stderr directly to files and retains native sessions during execution. -On success, failure, or cancellation, an independent Harbor exec requests launcher shutdown, stops the CLI process group, and scans Linux `/proc` for same-user descendants carrying the trial's unique environment marker. -Process start times protect against signaling a reused PID. -Cleanup escalates from TERM to KILL after five seconds, waits up to two more seconds, and records whether any live tracked processes remain. +On success, failure, or cancellation, an independent Harbor exec writes the stop barrier and signals identified launcher and CLI processes individually. +A private pipe prevents CLI startup until the launcher has recorded its PID and start time; launcher death before publication closes the pipe without starting agent execution. +No launcher process-group signal can include a retained service. +After control processes exit, cleanup reads durable ownership records and scans Linux `/proc` for trial-marked processes and identified shell groups. +Linux boot ID, PID, start time, process group, and shell execution tokens distinguish transferred groups from reused process IDs. +Environment-owned groups survive; active or otherwise untransferred work receives TERM, escalating to KILL within a seven-second recovery budget. +Control escalation begins after three seconds, reserving time for active descendants; active-work escalation begins after five seconds. Already exited zombies cannot execute and are left for sandbox init to reap. -A stop-request file prevents delayed launch after transport cancellation. -Task commands must preserve the inherited trial environment marker for detached descendants to remain discoverable. -Harbor must still delete the sandbox after the trial for complete environment isolation. +The report records `agent_execution_stopped`, `ownership_validated`, `remaining_agent_pids`, control identities, retained service identities, and the consumed ownership records. +Outcome parsing and gate validation reconcile these records with retained files. +`cleanup_confirmed` in the outcome means pre-verifier agent shutdown, not an empty sandbox. +Missing, malformed, contradictory, or unverified shutdown evidence raises `DiffexControlStopError`, which is outside Harbor's verifier-eligible timeout and nonzero-exit exceptions. +This exception overrides a pending cancellation while retaining its interrupted outcome in the evidence. +Verified shutdown preserves Harbor's existing verification behavior for deadlines and nonzero agent exits without changing their failed classifications. +Task commands must preserve the inherited trial marker or remain in an identified shell group for unfinished descendants to remain discoverable. +Harbor stops the shared environment after verification, including verifier failure or timeout recovery; separate-environment verification keeps Harbor's artifact-based lifecycle. +The benchmark runner independently confirms sandbox deletion after every trial, including abandonment and transport failures. Recovery is shielded against repeated cancellation and runs before `run()` returns or re-raises the interruption. It allows up to 15 seconds for the cleanup transport and 15 seconds for mounted-log preparation or 30 seconds for remote log download. diff --git a/evals/terminal-bench/RELEASE_GATE.md b/evals/terminal-bench/RELEASE_GATE.md index c96ca2f7a..eae9a1f63 100644 --- a/evals/terminal-bench/RELEASE_GATE.md +++ b/evals/terminal-bench/RELEASE_GATE.md @@ -2,7 +2,8 @@ This gate qualifies a specific Diffex adapter, dataset, model, and harness configuration for a larger benchmark run. It does not publish a product release or submit a leaderboard entry. -The implementation supports all qualification stages; live Diffex qualification remains pending a model-provider credential. +The implementation supports all qualification stages; live Diffex qualification requires provider credentials and a published release with environment-owned process support. +The checked-in Diffex 0.3.0 pin lacks that support and deliberately fails installation capability validation until a corrected release and integrity are pinned. ## Configuration and prerequisites @@ -102,7 +103,8 @@ The Harbor authentication and upload commands below also apply to the exported s The adapter runs from the local uv project, and a newly started command reads the current Python source and `benchmark.toml`. An already running command keeps its loaded code and policy; let it finish or interrupt it once with Ctrl-C and wait for cleanup before restarting. -No Diffex npm release or product rebuild is needed for these runner and policy changes; new sandboxes install the existing pinned release with the updated tool allowlist. +The environment-owned process policy requires a corrected Diffex npm release and a reviewed pin update before new sandboxes can qualify. +Development source images exercise the lifecycle without qualifying the installed release. After updating the checkout, run the following in your Mac's zsh terminal with `DAYTONA_API_KEY` and `OPENAI_API_KEY` exported. `git status --short` must produce no output. @@ -245,6 +247,12 @@ Diffex installation evidence includes the three CLI checks and verified npm/Node The oracle does not install Diffex or call a model, so its Diffex installation, model usage, and ATIF checks are not applicable. The gate re-parses native process/stream evidence instead of trusting the outcome summary alone. +The execution policy includes `background_process_lifetime: environment` in the requested/effective configuration and benchmark identity. +Pre-verifier evidence must establish that all agent execution stopped, with every retained task service backed by a durable environment ownership record. +The gate reconciles control identities, ownership files, cleanup reports, outcome summaries, and trajectory metadata. +Retained services do not count as cleanup failures, and verifier reward cannot convert a deadline or provider failure into completed execution. +An unverified control stop raises `DiffexControlStopError` and prevents Harbor from entering verification. +Final cleanup remains Harbor's environment stop followed by independent sandbox-deletion confirmation. It regenerates ATIF into a temporary directory, validates it with Harbor, and compares it with retained trajectories and image attachments. Native totals, per-model totals, `accounting.json`, and `AgentContext` must reconcile. Unknown telemetry remains unknown and cannot satisfy complete accounting. diff --git a/evals/terminal-bench/benchmark.toml b/evals/terminal-bench/benchmark.toml index be514ea9e..b829ae23e 100644 --- a/evals/terminal-bench/benchmark.toml +++ b/evals/terminal-bench/benchmark.toml @@ -30,3 +30,4 @@ thinking = "max" tools = "read,bash,edit,write,update_plan" subagents = "enabled" project_resources = "context-only" +background_process_lifetime = "environment" diff --git a/evals/terminal-bench/execution.yaml b/evals/terminal-bench/execution.yaml index d3008b3ac..e564b896a 100644 --- a/evals/terminal-bench/execution.yaml +++ b/evals/terminal-bench/execution.yaml @@ -13,3 +13,4 @@ agents: tools: read,bash,edit,write,update_plan subagents: enabled project_resources: context-only + background_process_lifetime: environment diff --git a/evals/terminal-bench/full.yaml b/evals/terminal-bench/full.yaml index 5152d17b3..5a80919d8 100644 --- a/evals/terminal-bench/full.yaml +++ b/evals/terminal-bench/full.yaml @@ -14,6 +14,7 @@ agents: tools: read,bash,edit,write,update_plan subagents: enabled project_resources: context-only + background_process_lifetime: environment env: OPENAI_API_KEY: ${OPENAI_API_KEY} datasets: diff --git a/evals/terminal-bench/scripts/smoke_outcomes.py b/evals/terminal-bench/scripts/smoke_outcomes.py index 5eadef6d3..345ea1ee4 100644 --- a/evals/terminal-bench/scripts/smoke_outcomes.py +++ b/evals/terminal-bench/scripts/smoke_outcomes.py @@ -18,15 +18,18 @@ from diffex_terminal_bench.adapter import DiffexAgent from diffex_terminal_bench.config import PINS +from diffex_terminal_bench.dataset import load_config, prepare_dataset +from diffex_terminal_bench.gate import sha256 from diffex_terminal_bench.outcome import DiffexExecutionError -async def smoke(platform): +async def smoke(platform, source_image=None, grpc_fixture=False): config = tomllib.loads(Path("smoke.toml").read_text()) session_id = f"diffex-outcomes-{uuid.uuid4().hex[:12]}" trial = TrialPaths(Path("results", session_id).resolve()) trial.mkdir() - report = {"platform": platform, "image": config["images"][platform], "pins": PINS, + report = {"platform": platform, "image": (source_image or config["images"][platform]), "pins": PINS, + "qualification": "source-only" if source_image else "pinned-release", "passed": False, "cleanup_passed": False, "outcomes": []} print(f"Outcome evidence: {trial.trial_dir}", flush=True) with tempfile.TemporaryDirectory(prefix="diffex-outcomes-") as temp: @@ -35,7 +38,7 @@ async def smoke(platform): overlay.write_text(f"services:\n main:\n platform: {platform}\n") environment = DockerEnvironment( environment_dir=directory, environment_name="diffex-outcomes-smoke", session_id=session_id, - trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=config["images"][platform]), + trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=(source_image or config["images"][platform])), extra_docker_compose=[overlay], mounts=[{"type": "bind", "source": str(trial.agent_dir), "target": "/logs/agent"}], ) @@ -43,12 +46,22 @@ async def smoke(platform): await asyncio.wait_for(environment.start(force_build=False), config["environment_timeout_sec"]) agent = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", extra_env={"OPENAI_API_KEY": "fixture-secret-never-log"}) - await asyncio.wait_for(agent.setup(environment), PINS["setup_timeout_sec"] + 15) + if source_image: + source_wrapper = directory / "source-cli" + source_wrapper.write_text('#!/bin/sh\nexport TSX_TSCONFIG_PATH=/repo/tsconfig.json\nexec node --import /repo/node_modules/tsx/dist/loader.mjs /repo/packages/coding-agent/src/cli.ts "$@"\n') + await environment.upload_file(source_wrapper, "/usr/local/bin/diffex") + await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex") + else: + await asyncio.wait_for(agent.setup(environment), PINS["setup_timeout_sec"] + 15) await environment.upload_file(Path("tests/faux-outcomes.cjs"), "/tmp/faux-outcomes.cjs") + await environment.upload_file(Path("src/diffex_terminal_bench/cleanup.cjs"), "/tmp/faux-cleanup.cjs") wrapper = directory / "diffex" wrapper.write_text( '#!/bin/sh\n' '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs prepare\n' + 'if [ "$(cat /tmp/outcome-scenario)" != interrupted-output ]; then\n' + ' exec /usr/local/bin/diffex-release "$@"\n' + 'fi\n' '/usr/local/bin/diffex-release "$@"\n' 'status=$?\n' '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs interrupt-output\n' @@ -59,17 +72,30 @@ async def smoke(platform): await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex && mkdir -p /task/work && chmod 777 /task/work") environment.task_env_config.workdir = "/task/work" await agent.exec_as_root(environment, command="nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs >/tmp/faux-outcomes.log 2>&1 dict: return {"adapter_commit": commit, "dataset": config.dataset.model_dump(), "task_digests": digests, "pins": PINS, "harbor_version": version("harbor"), "daytona_version": version("daytona"), "python_version": platform.python_version(), "uv_lock_sha256": sha256(PACKAGE / "uv.lock"), - "model": config.model, "policy": config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources"}), + "model": config.model, "policy": config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), "pilot_tasks": config.pilot_tasks, "backend": "daytona", "daytona_target": os.environ.get("DAYTONA_TARGET", "organization-default"), "daytona_api_url": os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api"), @@ -85,7 +85,7 @@ def trial_config(config: BenchmarkConfig, task_path: Path, directory: Path, labe agent = AgentConfig(name="oracle") if oracle else AgentConfig( import_path=DiffexAgent.import_path(), model_name=config.model, override_setup_timeout_sec=630, - kwargs=config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources"}), + kwargs=config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), # Variable references are serialized, never credential values. Harbor # also uses these names to scrub accidental credential output from logs. env={name: f"${{{name}}}" for name in PROVIDER_ENV[config.model.split("/", 1)[0]] if os.environ.get(name)}, diff --git a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs b/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs index fa33935a3..d58070256 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs @@ -19,60 +19,131 @@ function signal(pid, start, name) { try { process.kill(pid, name); } catch (error) { if (error.code !== "ESRCH") throw error; } } +function snapshot() { + const processes = new Map(); + for (const entry of readdirSync("/proc")) { + if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue; + try { + const stat = readFileSync(`/proc/${entry}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/); + if (fields[0] === "Z") continue; + const uid = Number(readFileSync(`/proc/${entry}/status`, "utf8").match(/^Uid:\s+(\d+)/m)?.[1]); + let env; + try { env = readFileSync(`/proc/${entry}/environ`, "utf8").split("\0"); } + catch (error) { + if (error.code !== "EACCES") throw error; + if (uid !== process.getuid()) continue; + } + processes.set(Number(entry), { pid: Number(entry), start: fields[19], group: Number(fields[2]), env }); + } catch (error) { + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } + } + return processes; +} + +function ownership(runDir) { + const directory = join(runDir, "ownership"); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter((name) => name.endsWith(".json")).map((name) => { + const record = JSON.parse(readFileSync(join(directory, name), "utf8")); + if (record.version !== 1 || typeof record.executionId !== "string" || !record.executionId || + name !== `${record.executionId}.json` || + !Number.isSafeInteger(record.processGroupId) || record.processGroupId <= 1 || + !Number.isSafeInteger(record.supervisorPid) || record.supervisorPid <= 1 || + !((record.owner === "environment" && record.state === "background") || + (record.owner === "session" && ["active", "terminated"].includes(record.state))) || + !Array.isArray(record.members) || (record.owner === "environment" && !record.members.length) || + record.members.some((member) => !Number.isSafeInteger(member.pid) || member.pid <= 1 || + member.processGroupId !== record.processGroupId || typeof member.startTime !== "string" || !member.startTime)) { + throw new Error("Invalid shell ownership record"); + } + return record; + }); +} + async function cleanup(runDir) { - // Prevent a delayed launcher from starting after transport cancellation. + // This barrier must precede discovery, including when the launcher has not started. writeFileSync(join(runDir, "stop-requested"), "", { mode: 0o600 }); - const marker = `DIFFEX_TB_RUN_DIR=${runDir}`; - let launcher; - if (existsSync(join(runDir, "launcher.json"))) launcher = JSON.parse(readFileSync(join(runDir, "launcher.json"), "utf8")); - if (launcher) signal(launcher.pid, launcher.start, "SIGTERM"); - const seen = new Map(); + const controls = new Map(); const deadline = Date.now() + 7000; + const boot = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const seen = new Map(); let remaining = []; + let retained = []; + let records = []; + let stopped = false; + let quietScans = 0; do { - const unreadable = new Map(); - for (const entry of readdirSync("/proc")) { - if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue; - try { - if (!identity(entry)) continue; - if (readFileSync(`/proc/${entry}/environ`, "utf8").split("\0").includes(marker)) { - const start = identity(entry); - if (start) seen.set(Number(entry), start); + // Re-read identities to close delayed spawn/publication races. Do not + // inspect handoffs until their writers have stopped publishing them. + for (const name of ["launcher", "cli"]) { + const path = join(runDir, `${name}.json`); + if (!existsSync(path)) continue; + const control = JSON.parse(readFileSync(path, "utf8")); + if (Object.keys(control).sort().join(",") !== "pid,start" || !Number.isSafeInteger(control.pid) || control.pid <= 1 || + typeof control.start !== "string" || !control.start) throw new Error("Invalid control identity"); + controls.set(name, control); + } + // Leave time for TERM-resistant active descendants after control shutdown. + const force = Date.now() >= deadline - 4000; + for (const control of controls.values()) signal(control.pid, control.start, force ? "SIGKILL" : "SIGTERM"); + remaining = [...controls.values()].filter((control) => identity(control.pid) === control.start).map((control) => control.pid); + if (!remaining.length) { + stopped = true; + records = ownership(runDir); + const processes = snapshot(); + const owned = new Map(); + for (const record of records) { + const members = [...processes.values()].filter((member) => member.group === record.processGroupId); + const known = members.some((member) => record.members.some((previous) => + previous.pid === member.pid && previous.startTime === `${boot}:${member.start}`) || + member.env?.includes(`DIFFEX_SHELL_EXECUTION_ID=${record.executionId}`)); + if (!known) continue; // A recycled group ID is not ownership evidence. + for (const member of members) { + if (owned.has(member.pid) && owned.get(member.pid).executionId !== record.executionId) { + throw new Error("Conflicting shell ownership records"); + } + if ([...controls.values()].some((control) => control.pid === member.pid && control.start === member.start)) { + throw new Error("Control process claimed by shell ownership"); + } + owned.set(member.pid, record); } - } catch (error) { - if (error.code === "EACCES") { - if (!identity(entry)) continue; - // Other sandbox users' processes are outside this trial launcher. - let status; - try { status = readFileSync(`/proc/${entry}/status`, "utf8"); } - catch (statusError) { if (statusError.code === "ENOENT") continue; throw statusError; } - if (Number(status.match(/^Uid:\s+(\d+)/m)?.[1]) !== process.getuid()) continue; - // An exiting process can revoke environ access before becoming - // a zombie. Retry until it exits or becomes readable; never - // claim cleanup while a same-user process remains uninspectable. - const start = identity(entry); - if (start) unreadable.set(Number(entry), start); - continue; + } + retained = []; + for (const member of processes.values()) { + const record = owned.get(member.pid); + if (record?.owner === "environment") { + retained.push({ pid: member.pid, startTime: `${boot}:${member.start}`, + processGroupId: member.group, executionId: record.executionId, owner: "environment" }); + } else if (record || member.env?.includes(`DIFFEX_TB_RUN_DIR=${runDir}`)) { + seen.set(member.pid, member.start); + } else if (!member.env) { + remaining.push(member.pid); // Never claim success with uninspectable same-user work. } - if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + for (const [pid, start] of seen) signal(pid, start, Date.now() >= deadline - 2000 ? "SIGKILL" : "SIGTERM"); + remaining.push(...[...seen].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid)); + // A parent can fork after /proc enumeration and before its signal. + // Scan again after all discovered active work has exited. + quietScans = remaining.length ? 0 : quietScans + 1; + if (quietScans >= 2) break; } - const force = Date.now() >= deadline - 2000; - for (const [pid, start] of seen) signal(pid, start, force ? "SIGKILL" : "SIGTERM"); - if (launcher && force) signal(launcher.pid, launcher.start, "SIGKILL"); - remaining = [...seen].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid); - remaining.push(...[...unreadable].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid)); - if (launcher && identity(launcher.pid) === launcher.start) remaining.push(launcher.pid); - if (remaining.length === 0) break; await delay(100); } while (Date.now() < deadline); - const report = { confirmed: remaining.length === 0, remaining_pids: remaining, tracked_processes: seen.size }; + const report = { + schema_version: 1, phase: "pre-verifier", stop_requested: true, + agent_execution_stopped: stopped && remaining.length === 0 && quietScans >= 2, + ownership_validated: stopped, remaining_agent_pids: [...new Set(remaining)], + control_processes: [...controls].map(([role, control]) => ({ role, ...control })), + retained_processes: retained, ownership_records: records, + }; writeFileSync(join(runDir, "cleanup.json"), JSON.stringify(report), { mode: 0o600 }); - if (!report.confirmed) process.exitCode = 1; + if (!report.agent_execution_stopped) process.exitCode = 1; } module.exports = { identity }; if (require.main === module) cleanup(process.argv[2]).catch((error) => { - writeFileSync(join(process.argv[2], "cleanup.json"), JSON.stringify({ confirmed: false, error: error.code ?? error.name, path: error.path }), { mode: 0o600 }); + writeFileSync(join(process.argv[2], "cleanup.json"), JSON.stringify({ schema_version: 1, phase: "pre-verifier", agent_execution_stopped: false, ownership_validated: false, error: error.code ?? error.name, path: error.path }), { mode: 0o600 }); process.exitCode = 1; }); diff --git a/evals/terminal-bench/src/diffex_terminal_bench/execution.py b/evals/terminal-bench/src/diffex_terminal_bench/execution.py index 10506fa86..057c5b77e 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/execution.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/execution.py @@ -25,6 +25,7 @@ class ExecutionOptions(InstalledAgentOptions): tools: str = "read,bash,edit,write,update_plan" subagents: Literal["disabled", "enabled"] = "disabled" project_resources: Literal["context-only"] = "context-only" + background_process_lifetime: Literal["environment"] = "environment" # Arbitrary native config and prompt templates could defeat trial isolation # or alter the benchmark instruction. Reject them in Harbor preflight too. config: None = None diff --git a/evals/terminal-bench/src/diffex_terminal_bench/gate.py b/evals/terminal-bench/src/diffex_terminal_bench/gate.py index fafad8f1d..c4342982b 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/gate.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/gate.py @@ -178,6 +178,7 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = agent = directory / "agent" installation = read_json(agent / "setup/installation.json") expected = {"verified": True, "harbor_version": PINS["harbor_version"], + "background_process_lifetime": "environment", "node_version": f"v{PINS['node_version']}", "diffex_version": PINS["diffex_version"], "npm_integrity": PINS["diffex_integrity"], "npm_registry": PINS["npm_registry"]} if any(installation.get(key) != value for key, value in expected.items()): @@ -195,14 +196,19 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = outcome = read_json(run / "outcome.json") if outcome.get("status") != "completed" or outcome.get("recovery_errors"): raise ValueError("Diffex did not complete cleanly") - if parse_outcome(run, launcher_exit_code=outcome.get("launcher_exit_code"))["status"] != "completed": + native_outcome = parse_outcome(run, launcher_exit_code=outcome.get("launcher_exit_code")) + if native_outcome["status"] != "completed": raise ValueError("Native execution evidence does not establish completion") + if any(outcome.get(key) != native_outcome.get(key) for key in ( + "agent_execution_stopped", "cleanup_confirmed", "retained_processes")): + raise ValueError("Outcome differs from process ownership evidence") requested, effective = read_json(run / "requested.json"), read_json(run / "effective.json") provider, model = identity["model"].split("/", 1) policy = identity["policy"] tools = policy["tools"].split(",") + (list(SUBAGENT_TOOLS) if policy["subagents"] == "enabled" else []) for key, value in {"provider": provider, "model": model, "thinking": policy["thinking"], "subagents": policy["subagents"], "project_resources": policy["project_resources"], + "background_process_lifetime": policy["background_process_lifetime"], "pins": PINS, "tools": tools, "evolution": {"enabled": False}, "state_policy": "fresh-per-run"}.items(): if requested.get(key) != value: raise ValueError(f"Requested configuration differs from gate identity: {key}") diff --git a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py new file mode 100644 index 000000000..6441f1236 --- /dev/null +++ b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py @@ -0,0 +1,74 @@ +"""Validate pre-verifier control shutdown separately from environment teardown.""" + +import json +from pathlib import Path + + +def read_cleanup(directory: Path) -> dict: + report = json.loads((directory / "cleanup.json").read_text()) + if (not isinstance(report, dict) or type(report.get("schema_version")) is not int or report["schema_version"] != 1 + or report.get("phase") != "pre-verifier" or report.get("stop_requested") is not True + or report.get("agent_execution_stopped") is not True + or report.get("ownership_validated") is not True + or report.get("remaining_agent_pids") != [] or not (directory / "stop-requested").is_file()): + raise ValueError("Agent execution stop is unverified") + controls = report.get("control_processes") + records = report.get("ownership_records") + retained = report.get("retained_processes") + if not all(isinstance(value, list) for value in (controls, records, retained)): + raise ValueError("Missing ownership or control evidence") + expected = [] + for role in ("launcher", "cli"): + path = directory / f"{role}.json" + if path.exists(): + control = json.loads(path.read_text()) + if (not isinstance(control, dict) or set(control) != {"pid", "start"} + or type(control.get("pid")) is not int or control["pid"] <= 1 + or not isinstance(control.get("start"), str) or not control["start"]): + raise ValueError("Invalid control identity") + expected.append({"role": role, **control}) + if controls != expected: + raise ValueError("Control identities differ from shutdown evidence") + process_path = directory / "process.json" + if process_path.exists(): + process = json.loads(process_path.read_text()) + roles = {control["role"] for control in controls} + if "launcher" not in roles or (type(process.get("exit_code")) is int + and not process.get("spawn_error") and "cli" not in roles): + raise ValueError("Missing launched control identity") + saved = [] + for path in sorted((directory / "ownership").glob("*.json")): + record = json.loads(path.read_text()) + if path.name != f"{record['executionId']}.json": + raise ValueError("Ownership filename differs from execution identity") + saved.append(record) + if sorted(records, key=lambda record: record["executionId"]) != sorted(saved, key=lambda record: record["executionId"]): + raise ValueError("Ownership records differ from shutdown evidence") + by_id = {} + for record in records: + if (type(record.get("version")) is not int or record["version"] != 1 or not isinstance(record.get("executionId"), str) + or not record["executionId"] or record["executionId"] in by_id + or type(record.get("processGroupId")) is not int or record["processGroupId"] <= 1 + or type(record.get("supervisorPid")) is not int or record["supervisorPid"] <= 1 + or (record.get("owner"), record.get("state")) not in ( + ("environment", "background"), ("session", "active"), ("session", "terminated")) + or not isinstance(record.get("members"), list) + or (record["owner"] == "environment" and not record["members"])): + raise ValueError("Invalid ownership record") + for member in record["members"]: + if (type(member.get("pid")) is not int or member["pid"] <= 1 + or member.get("processGroupId") != record["processGroupId"] + or not isinstance(member.get("startTime"), str) or not member["startTime"]): + raise ValueError("Invalid owned process identity") + by_id[record["executionId"]] = record + seen = set() + for member in retained: + record = by_id.get(member.get("executionId"), {}) + if (member.get("owner") != "environment" or record.get("owner") != "environment" + or member.get("processGroupId") != record.get("processGroupId") + or type(member.get("pid")) is not int or member["pid"] <= 1 or member["pid"] in seen + or not isinstance(member.get("startTime"), str) or not member["startTime"] + or any(control["pid"] == member["pid"] for control in controls)): + raise ValueError("Retained process lacks unambiguous environment ownership") + seen.add(member["pid"]) + return report diff --git a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs index 43a5e1f95..cb2257694 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs @@ -23,6 +23,8 @@ export default function observe(api) { provider: ctx.model?.provider, model: ctx.model?.id, thinking: api.getThinkingLevel(), tools: api.getActiveTools(), cwd: ctx.cwd, root_session: rootSession, + background_process_lifetime: process.argv.includes("--background-process-lifetime") + ? process.argv[process.argv.indexOf("--background-process-lifetime") + 1] : null, instruction_transport: "native-file", instruction_delivered: prompt === `\n${instruction}\n\n`, instruction_sha256: createHash("sha256").update(instruction).digest("hex"), diff --git a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py index c1ae60304..1c506f8f0 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py @@ -6,11 +6,17 @@ from harbor.agents.installed.base import AgentAuthenticationError, NonZeroAgentExitCodeError +from diffex_terminal_bench.lifecycle import read_cleanup + class DiffexExecutionError(NonZeroAgentExitCodeError): """An agent infrastructure failure, not a completed task with reward zero.""" +class DiffexControlStopError(RuntimeError): + """Block Harbor verification when agent execution may still be running.""" + + def parse_outcome(directory: Path, interruption: str | None = None, launcher_exit_code: int | None = None) -> dict: result = {"status": "incomplete", "reason": "missing_evidence", "stream_complete": False, @@ -21,12 +27,14 @@ def parse_outcome(directory: Path, interruption: str | None = None, if not isinstance(process, dict): raise ValueError("invalid_process") result["process"] = process - cleanup = json.loads((directory / "cleanup.json").read_text()) - if not isinstance(cleanup, dict): - raise ValueError("invalid_cleanup") - result["cleanup_confirmed"] = cleanup.get("confirmed") is True except (OSError, ValueError): process = {} + try: + cleanup = read_cleanup(directory) + result.update(cleanup_confirmed=True, agent_execution_stopped=True, + retained_processes=cleanup["retained_processes"]) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + result.update(cleanup_confirmed=False, agent_execution_stopped=False) # Parse incrementally so long runs do not require the whole stream in memory. active = False ended = False @@ -132,7 +140,8 @@ def parse_outcome(directory: Path, interruption: str | None = None, effective = json.loads((directory / "effective.json").read_text()) if not isinstance(requested, dict) or not isinstance(effective, dict): raise ValueError("invalid_configuration") - matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256")) + matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime")) + matches = matches and requested.get("background_process_lifetime") == "environment" matches = matches and effective.get("release_agents_loaded") is True and effective.get("instruction_delivered") is True matches = matches and sorted(effective["tools"]) == sorted(requested["tools"]) except (OSError, ValueError, KeyError, TypeError): @@ -143,6 +152,8 @@ def parse_outcome(directory: Path, interruption: str | None = None, def require_completed(outcome: dict) -> None: + if outcome.get("agent_execution_stopped") is not True: + raise DiffexControlStopError("Agent execution stop is unverified; verification is blocked") if outcome["status"] == "completed": return message = f"Diffex execution {outcome['status']}: {outcome['reason']}; see retained run artifacts" diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs index c51bfd5ec..0421b896f 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs @@ -40,6 +40,8 @@ const args = [ "--thinking", process.env.DIFFEX_TB_THINKING, "--tools", config.tools.join(","), "--session", join(runDir, "sessions", "root.jsonl"), "--session-dir", join(runDir, "sessions"), + "--background-process-lifetime", config.background_process_lifetime, + "--background-process-ownership-dir", join(runDir, "ownership"), "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--extension", join(runDir, "observe.mjs"), "--print", `@${join(runDir, "instruction.txt")}`, ]; @@ -49,9 +51,16 @@ const stderr = openSync(join(runDir, "stderr.log"), "wx", 0o600); writeFileSync(join(runDir, "process.json"), JSON.stringify({ exit_code: null, signal: null, spawn_error: null, interrupted: false, }), { mode: 0o600 }); -const child = spawn("/usr/local/bin/diffex", args, { - cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr], detached: true, +// The child cannot execute the CLI before its identity is durable. If the +// launcher dies during publication, EOF on the private pipe closes the gate. +// exec preserves this PID and start time through the installed CLI wrapper. +const child = spawn("/bin/sh", ["-c", + 'read -r ready <&3 && [ "$ready" = start ] || exit 1; exec 3<&-; exec /usr/local/bin/diffex "$@"', + "diffex", ...args], { + cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr, "pipe"], detached: true, }); +const childStart = child.pid ? identity(child.pid) : null; +if (child.pid) writeFileSync(join(runDir, "cli.json"), JSON.stringify({ pid: child.pid, start: childStart }), { mode: 0o600 }); closeSync(stdout); closeSync(stderr); let spawnError = null; @@ -61,13 +70,20 @@ function stop() { if (interrupted) return; interrupted = true; if (!child.pid) return; - try { process.kill(-child.pid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; } + // Signal only the identified CLI. Group signals could reach retained task + // services; independent recovery handles active descendants after CLI exit. + if (identity(child.pid) === childStart) { + try { process.kill(child.pid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; } + } killTimer = setTimeout(() => { - try { process.kill(-child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + if (identity(child.pid) === childStart) { + try { process.kill(child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + } }, 5000); } for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, stop); child.on("error", (error) => { spawnError = error.code ?? error.name; }); +child.stdio[3].on("error", (error) => { spawnError = error.code ?? error.name; }); child.on("close", (code, signal) => { clearTimeout(killTimer); writeFileSync(join(runDir, "process.json"), JSON.stringify({ @@ -76,4 +92,9 @@ child.on("close", (code, signal) => { process.exitCode = interrupted ? 1 : (code ?? 1); }); // Close the race with a stop request written during synchronous initialization. -if (existsSync(join(runDir, "stop-requested"))) stop(); +if (existsSync(join(runDir, "stop-requested"))) { + child.stdio[3].end(); + stop(); +} else { + child.stdio[3].end("start\n"); +} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py b/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py index 3ba9a1b29..875216d12 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py @@ -400,7 +400,10 @@ def aggregate(document): accounting = {"complete": not gaps and not missing, "gaps": sorted(set(gaps)), "missing_fields": sorted(missing), "observed_usage": observed, "observed_model_usage": dict(by_model), "totals": totals} - root_document["extra"].update(accounting=accounting, execution_status=outcome.get("status")) + root_document["extra"].update(accounting=accounting, execution_status=outcome.get("status"), + background_process_lifetime=requested.get("background_process_lifetime"), + agent_execution_stopped=outcome.get("agent_execution_stopped"), + retained_processes=outcome.get("retained_processes", [])) root_document["final_metrics"] = { **{f"total_{field}": value for field, value in totals.items()}, "total_steps": len(root.steps), "extra": {"scope": "root_and_all_descendants", "observed_usage": observed, "accounting_complete": accounting["complete"]}, diff --git a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs b/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs index 7918430c2..5fa6e9537 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs @@ -23,10 +23,15 @@ assert.equal(installed.resolved, tarball, "Installed registry mismatch"); if (process.argv[2] === "installed") process.exit(0); assert.equal(process.argv[2], "complete"); assert.equal(readFileSync(join(logs, "version.stdout.txt"), "utf8").trim(), env.DIFFEX_VERSION); +const help = readFileSync(join(logs, "help.stdout.txt"), "utf8"); +for (const flag of ["--background-process-lifetime", "--background-process-ownership-dir"]) { + assert.ok(help.includes(flag), `Pinned Diffex release lacks ${flag}; publish and pin the corrected release`); +} assert.ok(Number(env.SETUP_ELAPSED_SEC) < Number(env.SETUP_TIMEOUT_SEC), "Setup exceeded deadline"); writeFileSync(join(logs, "installation.json"), `${JSON.stringify({ schema_version: 1, verified: true, + background_process_lifetime: "environment", started_at: env.SETUP_STARTED_AT, elapsed_sec: Number(env.SETUP_ELAPSED_SEC), setup_timeout_sec: Number(env.SETUP_TIMEOUT_SEC), diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs new file mode 100644 index 000000000..6d624a8aa --- /dev/null +++ b/evals/terminal-bench/tests/cleanup.test.cjs @@ -0,0 +1,171 @@ +// Run in a Linux container: node --test /eval/tests/cleanup.test.cjs. +const assert = require("node:assert/strict"); +const { spawn } = require("node:child_process"); +const { once } = require("node:events"); +const { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } = require("node:fs"); +const { createServer, connect } = require("node:net"); +const { tmpdir } = require("node:os"); +const { join } = require("node:path"); +const { setTimeout: delay } = require("node:timers/promises"); +const { test } = require("node:test"); +const { identity } = require("../src/diffex_terminal_bench/cleanup.cjs"); + +if (process.argv[2] === "service") { + process.on("SIGTERM", () => {}); + const server = createServer((socket) => socket.end(String(process.pid))); + server.listen(0, "127.0.0.1", () => process.send(server.address().port)); +} else { + async function fixture(t) { + const runDir = mkdtempSync(join(tmpdir(), "diffex-cleanup-")); + mkdirSync(join(runDir, "ownership")); + const children = []; + t.after(async () => { + for (const child of children) { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await once(child, "exit"); + } + } + rmSync(runDir, { recursive: true, force: true }); + }); + return { + runDir, + async service(executionId, marked = true) { + const child = spawn(process.execPath, [__filename, "service"], { + detached: true, stdio: ["ignore", "ignore", "inherit", "ipc"], + env: { ...process.env, ...(marked ? { DIFFEX_TB_RUN_DIR: runDir } : {}), + DIFFEX_SHELL_EXECUTION_ID: executionId }, + }); + children.push(child); + const [port] = await once(child, "message", { signal: t.signal }); + const start = identity(child.pid); + const member = { pid: child.pid, processGroupId: child.pid, + startTime: `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` }; + return { child, port, start, member, executionId }; + }, + publish(service, owner = "environment", overrides = {}) { + const record = { version: 1, executionId: service.executionId, owner, + state: owner === "environment" ? "background" : "active", processGroupId: service.child.pid, + supervisorPid: process.pid, members: [service.member], ...overrides }; + writeFileSync(join(runDir, `ownership/${service.executionId}.json`), JSON.stringify(record)); + return record; + }, + async cleanup(expectedCode = 0) { + const child = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/cleanup.cjs"), runDir], { stdio: "inherit" }); + children.push(child); + assert.equal((await once(child, "exit", { signal: t.signal }))[0], expectedCode); + return JSON.parse(readFileSync(join(runDir, "cleanup.json"), "utf8")); + }, + }; + } + + async function verify(service) { + assert.equal(identity(service.child.pid), service.start, "the original service must survive cleanup"); + const socket = connect(service.port, "127.0.0.1"); + try { + assert.equal(String((await once(socket, "data"))[0]), String(service.child.pid)); + } finally { + socket.destroy(); + } + } + + const options = { skip: process.platform !== "linux", timeout: 15000 }; + test("pre-verifier cleanup preserves a transferred marked service", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + await verify(service); + const report = await f.cleanup(); + assert.equal(report.agent_execution_stopped, true); + assert.deepEqual(report.remaining_agent_pids, []); + assert.equal(report.retained_processes[0].pid, service.child.pid); + await verify(service); + assert.deepEqual(await f.cleanup(), report, "cleanup is idempotent"); + }); + + test("forced CLI death preserves handoffs and kills TERM-resistant unfinished work", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + const active = await f.service("active"); + f.publish(active, "session"); + const cli = await f.service("control"); + writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); + cli.child.kill("SIGKILL"); + await once(cli.child, "exit"); + const report = await f.cleanup(); + assert.equal(identity(active.child.pid), null); + assert.equal(report.control_processes[0].pid, cli.child.pid); + await verify(service); + }); + + test("cleanup stops TERM-resistant control activity before allowing verification", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service); + const cli = await f.service("control"); + writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); + const report = await f.cleanup(); + assert.equal(report.agent_execution_stopped, true); + assert.equal(identity(cli.child.pid), null); + await verify(service); + }); + + test("PID reuse cannot signal an unrelated process or authorize a stale handoff", options, async (t) => { + const f = await fixture(t); + const unrelated = await f.service("unrelated", false); + writeFileSync(join(f.runDir, "launcher.json"), JSON.stringify({ pid: unrelated.child.pid, start: "recycled" })); + const active = await f.service("active"); + f.publish(active, "environment", { executionId: "stale", members: [{ ...active.member, startTime: "old-boot:0" }] }); + // The filename and token must refer to the stale execution, not the new one. + const stale = join(f.runDir, "ownership/active.json"); + writeFileSync(join(f.runDir, "ownership/stale.json"), readFileSync(stale)); + rmSync(stale); + const report = await f.cleanup(); + assert.deepEqual(report.retained_processes, []); + assert.equal(identity(active.child.pid), null); + await verify(unrelated); + }); + + test("invalid or contradictory handoff evidence fails closed without killing the service", options, async (t) => { + const f = await fixture(t); + const service = await f.service("service"); + f.publish(service, "environment", { state: "active" }); + assert.equal((await f.cleanup(1)).agent_execution_stopped, false); + await verify(service); + f.publish(service); + writeFileSync(join(f.runDir, "ownership/conflict.json"), JSON.stringify({ + ...f.publish(service), executionId: "conflict", owner: "session", state: "active", + })); + assert.equal((await f.cleanup(1)).agent_execution_stopped, false); + await verify(service); + }); + + test("stop barrier prevents a delayed real launcher from spawning the CLI", options, async (t) => { + const f = await fixture(t); + await f.cleanup(); + const launcher = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir]); + assert.equal((await once(launcher, "exit", { signal: t.signal }))[0], 1); + const report = await f.cleanup(); + assert.equal(report.control_processes[0].role, "launcher"); + assert.equal(report.agent_execution_stopped, true); + }); + + test("launcher death before PID publication cannot release the CLI gate", options, async (t) => { + const f = await fixture(t); + writeFileSync(join(f.runDir, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", + thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment" })); + writeFileSync(join(f.runDir, "release-agents.md"), "Fixture context"); + const launcher = spawn(process.execPath, ["--require", join(__dirname, "fixtures/kill-before-cli-publication.cjs"), + join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir], { + env: { ...process.env, DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "off" }, + }); + assert.equal((await once(launcher, "exit", { signal: t.signal }))[1], "SIGKILL"); + const child = JSON.parse(readFileSync(join(f.runDir, "gated-child.json"), "utf8")); + // EOF must stop the waiting child even before independent cleanup runs. + const deadline = Date.now() + 2000; + while (identity(child.pid) === child.start && Date.now() < deadline) await delay(10); + assert.notEqual(identity(child.pid), child.start); + assert.equal((await f.cleanup()).agent_execution_stopped, true); + }); +} diff --git a/evals/terminal-bench/tests/faux-outcomes.cjs b/evals/terminal-bench/tests/faux-outcomes.cjs index 1d94db012..80f7940ee 100644 --- a/evals/terminal-bench/tests/faux-outcomes.cjs +++ b/evals/terminal-bench/tests/faux-outcomes.cjs @@ -1,11 +1,43 @@ // Outcome smoke fixture. All model responses stay on container loopback. const { createServer } = require("node:http"); -const { spawn } = require("node:child_process"); +const { createServer: tcpServer, connect } = require("node:net"); +const assert = require("node:assert/strict"); +const { once } = require("node:events"); +const { setTimeout: delay } = require("node:timers/promises"); const { readFileSync, writeFileSync } = require("node:fs"); const { join, basename } = require("node:path"); +const { identity } = require("./faux-cleanup.cjs"); + const action = process.argv[2]; -if (action === "prepare") { +if (action === "health") { + (async () => { + for (let attempt = 0; attempt < 20; attempt++) { + try { + if ((await fetch("http://127.0.0.1:8766/health", { signal: AbortSignal.timeout(1000) })).ok) return; + } catch {} + await delay(100); + } + process.exitCode = 1; + })(); +} else if (action === "verify-service") { + (async () => { + const runDir = process.argv[3]; + const service = JSON.parse(readFileSync(join(runDir, "service.json"), "utf8")); + assert.equal(identity(service.pid), service.start); + for (const name of ["launcher", "cli"]) { + const control = JSON.parse(readFileSync(join(runDir, `${name}.json`), "utf8")); + assert.notEqual(identity(control.pid), control.start); + } + const socket = connect(service.port, "127.0.0.1"); + if (service.protocol === "grpc") await once(socket, "connect"); + else assert.equal(String((await once(socket, "data"))[0]), String(service.pid)); + socket.destroy(); + const provider = await (await fetch(`http://127.0.0.1:8766/state/${basename(runDir)}`)).json(); + assert.equal(provider.active, 0, "provider activity must stop before verification"); + console.log(JSON.stringify({ ...service, provider, verified: true })); + })().catch((error) => { console.error(error); process.exitCode = 1; }); +} else if (action === "prepare") { const state = process.env.DIFFEX_CODING_AGENT_DIR; const runDir = process.env.DIFFEX_TB_RUN_DIR; const scenario = readFileSync("/tmp/outcome-scenario", "utf8").trim(); @@ -17,8 +49,7 @@ if (action === "prepare") { }], } } })); process.exit(0); -} -if (action === "interrupt-output") { +} else if (action === "interrupt-output") { const runDir = process.env.DIFFEX_TB_RUN_DIR; if (readFileSync("/tmp/outcome-scenario", "utf8").trim() === "interrupted-output") { const path = join(runDir, "stdout.jsonl"); @@ -27,34 +58,66 @@ if (action === "interrupt-output") { writeFileSync(path, lines.filter((line) => JSON.parse(line).type !== "agent_end").join("\n") + '\n{"type":'); } process.exit(0); -} -if (action === "linger") { - // A detached tool descendant deliberately ignores TERM. +} else if (action === "linger") { + // Successful shell completion transfers this TERM-resistant service. process.on("SIGTERM", () => {}); - writeFileSync(join(process.env.DIFFEX_TB_RUN_DIR, "lingering-pid"), String(process.pid)); - setInterval(() => {}, 1000); -} else if (action === "spawn-linger") { - const child = spawn(process.execPath, [__filename, "linger"], { detached: true, stdio: "ignore" }); - child.unref(); + const runDir = process.env.DIFFEX_TB_RUN_DIR; + const server = tcpServer((socket) => socket.end(String(process.pid))); + server.listen(0, "127.0.0.1", () => writeFileSync(join(runDir, "service.json"), JSON.stringify({ + pid: process.pid, start: identity(process.pid), port: server.address().port, + }))); } else { const counts = new Map(); + const states = new Map(); createServer(async (request, response) => { if (request.url === "/health") { response.end("ok"); return; } + if (request.url.startsWith("/state/")) { + response.end(JSON.stringify(states.get(request.url.split("/")[2]))); + return; + } let text = ""; for await (const chunk of request) text += chunk; const body = JSON.parse(text); const scenario = request.url.split("/")[1]; const count = (counts.get(request.url) ?? 0) + 1; counts.set(request.url, count); + const run = request.url.split("/")[2]; + const state = states.get(run) ?? { requests: 0, active: 0 }; + states.set(run, state); + state.requests++; + state.active++; + response.on("close", () => state.active--); + if (["success", "service-failure", "timeout", "forced-death", "grpc"].includes(scenario) && count === 2) { + const runDir = join("/logs/agent", run); + const service = JSON.parse(readFileSync(join(runDir, "service.json"), "utf8")); + const socket = connect(service.port, "127.0.0.1"); + await once(socket, "connect"); + socket.destroy(); + writeFileSync(join(runDir, "service-before-exit.json"), JSON.stringify(service)); + if (scenario === "forced-death") { + const cli = JSON.parse(readFileSync(join(runDir, "cli.json"), "utf8")); + assert.equal(identity(cli.pid), cli.start); + process.kill(cli.pid, "SIGKILL"); + response.destroy(); + return; + } + } + if (scenario === "service-failure" && count > 1) { + response.writeHead(400, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ error: { message: "Controlled failure after handoff" } })); + return; + } if (scenario === "authentication" || scenario === "exhausted" || (scenario === "recovered" && count === 1)) { response.writeHead(scenario === "authentication" ? 401 : 503, { "Content-Type": "application/json" }); response.end(JSON.stringify({ error: { message: scenario === "authentication" ? "Controlled invalid API key" : "Controlled service overloaded" } })); return; } if (scenario === "timeout" && count > 1) return; - const tool = count === 1 && ["timeout", "success"].includes(scenario) - ? { name: "bash", arguments: JSON.stringify({ command: "/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs spawn-linger; printf retained-diagnostic >&2; printf tool-ok" }) } + let tool = count === 1 && ["timeout", "success", "service-failure", "forced-death"].includes(scenario) + ? { name: "bash", arguments: JSON.stringify({ command: 'nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs linger >/dev/null 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done; printf tool-ok' }) } : null; + if (scenario === "grpc" && count === 1) tool = { name: "bash", arguments: JSON.stringify({ command: + 'cd /app && /opt/verifier/bin/python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. kv-store.proto && nohup /opt/verifier/bin/python /app/server.py >/app/server.log 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done' }) }; response.writeHead(200, { "Content-Type": "text/event-stream" }); const send = (delta, finish_reason = null) => response.write(`data: ${JSON.stringify({ id: `fixture-${count}`, object: "chat.completion.chunk", model: body.model, diff --git a/evals/terminal-bench/tests/fixtures/Dockerfile.source b/evals/terminal-bench/tests/fixtures/Dockerfile.source new file mode 100644 index 000000000..1aff9db77 --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/Dockerfile.source @@ -0,0 +1,11 @@ +FROM node:22.23.2-bookworm-slim@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9 +WORKDIR /repo +COPY package.json package-lock.json tsconfig.json tsconfig.base.json ./ +COPY packages ./packages +RUN npm ci --ignore-scripts --no-audit --no-fund +RUN npm run hydrate:model-data +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && sed -i "s|http://deb.debian.org|https://deb.debian.org|g" /etc/apt/sources.list.d/debian.sources && apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv git procps && rm -rf /var/lib/apt/lists/* +RUN python3 -m venv /opt/verifier && /opt/verifier/bin/pip install pytest==8.4.2 grpcio==1.75.1 grpcio-tools==1.75.1 +RUN mkdir -p /opt/diffex-terminal-bench/node/bin /opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist && ln -s /usr/local/bin/node /opt/diffex-terminal-bench/node/bin/node +ENV TSX_TSCONFIG_PATH=/repo/tsconfig.json +WORKDIR /task diff --git a/evals/terminal-bench/tests/fixtures/grpc_server.py b/evals/terminal-bench/tests/fixtures/grpc_server.py new file mode 100644 index 000000000..cc8d422a9 --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/grpc_server.py @@ -0,0 +1,37 @@ +"""Known-correct service for the pinned verifier, with process identity evidence.""" + +from concurrent import futures +import json +import os +from pathlib import Path +import threading + +import grpc +import kv_store_pb2 +import kv_store_pb2_grpc + + +class Server(kv_store_pb2_grpc.KVStoreServicer): + def __init__(self): + self.values = {} + self.lock = threading.Lock() + + def GetVal(self, request, context): + with self.lock: + return kv_store_pb2.GetValResponse(val=self.values.get(request.key, 0)) + + def SetVal(self, request, context): + with self.lock: + self.values[request.key] = request.value + return kv_store_pb2.SetValResponse(val=request.value) + + +server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) +kv_store_pb2_grpc.add_KVStoreServicer_to_server(Server(), server) +assert server.add_insecure_port("127.0.0.1:5328") == 5328 +server.start() +stat = Path(f"/proc/{os.getpid()}/stat").read_text().rsplit(")", 1)[1].split() +Path(os.environ["DIFFEX_TB_RUN_DIR"], "service.json").write_text(json.dumps({ + "pid": os.getpid(), "start": stat[19], "port": 5328, "protocol": "grpc", +})) +server.wait_for_termination() diff --git a/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs b/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs new file mode 100644 index 000000000..b7228e47d --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs @@ -0,0 +1,12 @@ +// Terminate the real launcher at its spawn-to-publication boundary. +const fs = require("node:fs"); +const { basename, dirname, join } = require("node:path"); + +const write = fs.writeFileSync; +fs.writeFileSync = (path, ...args) => { + if (basename(path) === "cli.json") { + write(join(dirname(path), "gated-child.json"), ...args); + process.kill(process.pid, "SIGKILL"); + } + return write(path, ...args); +}; diff --git a/evals/terminal-bench/tests/fixtures/kv-store.proto b/evals/terminal-bench/tests/fixtures/kv-store.proto new file mode 100644 index 000000000..6e401bc3f --- /dev/null +++ b/evals/terminal-bench/tests/fixtures/kv-store.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +service KVStore { + rpc GetVal (GetValRequest) returns (GetValResponse) {} + rpc SetVal (SetValRequest) returns (SetValResponse) {} +} +message GetValRequest { string key = 1; } +message GetValResponse { int32 val = 1; } +message SetValRequest { string key = 1; int32 value = 2; } +message SetValResponse { int32 val = 1; } diff --git a/evals/terminal-bench/tests/test_execution.py b/evals/terminal-bench/tests/test_execution.py index 257a9fbe1..a03fe9dd1 100644 --- a/evals/terminal-bench/tests/test_execution.py +++ b/evals/terminal-bench/tests/test_execution.py @@ -26,7 +26,7 @@ def setUp(self): # and a Linux smoke using the real launcher and provider. async def finish_run(agent, environment, run_dir, context, execution): (agent.logs_dir / run_dir.name).mkdir(parents=True, exist_ok=True) - return {"status": "completed"} + return {"status": "completed", "agent_execution_stopped": True} finish = patch.object(DiffexAgent, "_finish_run", autospec=True, side_effect=finish_run) finish.start() @@ -86,6 +86,7 @@ async def upload(source, target): self.assertEqual(configs[0]["credential_env_names"], ["OPENAI_API_KEY"]) self.assertNotIn("spawn_agent", configs[0]["tools"]) self.assertEqual(configs[0]["evolution"], {"enabled": False}) + self.assertEqual(configs[0]["background_process_lifetime"], "environment") calls = [call.kwargs for call in environment.exec.call_args_list if call.kwargs.get("env")] self.assertEqual(len(calls), 2) for call in calls: @@ -147,7 +148,7 @@ async def test_missing_credentials_and_ambiguous_models_fail_before_exec(self): environment.exec.assert_not_called() def test_preflight_rejects_implicit_or_unsafe_policy(self): - for kwargs in ({"thinking": "automatic"}, {"tools": "bash,bash"}, {"tools": "spawn_agent"}, + for kwargs in ({"background_process_lifetime": "session"}, {"thinking": "automatic"}, {"tools": "bash,bash"}, {"tools": "spawn_agent"}, {"project_resources": "all"}, {"subagents": True}, {"unknown": "option"}, {"config": {"evolution": {"enabled": True}}}, {"prompt_template_path": "template.txt"}): with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): diff --git a/evals/terminal-bench/tests/test_gate.py b/evals/terminal-bench/tests/test_gate.py index f55516bdd..d9802d394 100644 --- a/evals/terminal-bench/tests/test_gate.py +++ b/evals/terminal-bench/tests/test_gate.py @@ -20,6 +20,7 @@ from diffex_terminal_bench.gate import inventory, require_previous, sha256, stage_tasks, validate_trial, write_json from diffex_terminal_bench.outcome import parse_outcome from diffex_terminal_bench.trajectory import export_trajectory +from test_outcome import retain_service, write_cleanup from test_trajectory import USAGE, assistant, entry, interrupted_child_records @@ -32,10 +33,10 @@ def setUp(self): self.run = self.directory / "agent/run-fixture" self.run.mkdir(parents=True) self.identity = {"model": "openai/gpt-5", "policy": {"thinking": "high", "tools": "bash", - "subagents": "disabled", "project_resources": "context-only"}, + "subagents": "disabled", "project_resources": "context-only", "background_process_lifetime": "environment"}, "release_agents_sha256": "release-hash", "task_digests": {"task": "sha256:task"}} requested = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], - "subagents": "disabled", "project_resources": "context-only", "pins": PINS, + "subagents": "disabled", "project_resources": "context-only", "background_process_lifetime": "environment", "pins": PINS, "evolution": {"enabled": False}, "state_policy": "fresh-per-run"} (self.run / "instruction.txt").write_text("task") (self.run / "system-prompt.txt").write_text("system") @@ -44,7 +45,7 @@ def setUp(self): write_json(self.run / "effective.json", {**requested, "release_agents_loaded": True, "instruction_delivered": True, "release_agents_sha256": "release-hash", "system_prompt_sha256": sha256(self.run / "system-prompt.txt")}) write_json(self.run / "process.json", {"exit_code": 0}) - write_json(self.run / "cleanup.json", {"confirmed": True}) + write_cleanup(self.run) message = assistant("response") events = [{"type": "session"}, {"type": "agent_start"}, {"type": "message_start", "message": message}, {"type": "message_end", "message": message}, {"type": "agent_end", "messages": [message]}, @@ -61,6 +62,7 @@ def setUp(self): (self.run / name).touch() setup = self.directory / "agent/setup" write_json(setup / "installation.json", {"verified": True, "harbor_version": PINS["harbor_version"], + "background_process_lifetime": "environment", "node_version": f"v{PINS['node_version']}", "diffex_version": PINS["diffex_version"], "npm_integrity": PINS["diffex_integrity"], "npm_registry": PINS["npm_registry"], "elapsed_sec": 10}) for name in ("version", "help", "list-models"): @@ -286,6 +288,18 @@ def test_zero_reward_is_valid_benchmark_result(self): self.assertEqual(report["rewards"], {"reward": 0}) self.assertEqual(report["usage"]["prompt_tokens"], 19) + def test_service_handoff_qualifies_and_corrupt_ownership_cannot(self): + retain_service(self.run) + write_json(self.run / "outcome.json", parse_outcome(self.run, launcher_exit_code=0)) + self.reexport() + self.assertEqual(validate_trial(self.directory, "task", self.identity)["rewards"], {"reward": 0}) + self.mutate(self.run / "outcome.json", "retained_processes", []) + with self.assertRaisesRegex(ValueError, "ownership evidence"): + validate_trial(self.directory, "task", self.identity) + self.mutate(self.run / "ownership/service.json", "owner", "session") + with self.assertRaisesRegex(ValueError, "Native execution evidence"): + validate_trial(self.directory, "task", self.identity) + def test_missing_logs_and_forged_success_cannot_qualify(self): cases = [("agent/run-fixture/stderr.log", None, None), ("agent/run-fixture/process.json", "exit_code", 1), diff --git a/evals/terminal-bench/tests/test_installation.py b/evals/terminal-bench/tests/test_installation.py index 8dcde2a8a..bc965e9d4 100644 --- a/evals/terminal-bench/tests/test_installation.py +++ b/evals/terminal-bench/tests/test_installation.py @@ -53,6 +53,34 @@ async def test_execution_requires_explicit_model(self): class RegistryIntegrityTests(unittest.TestCase): + def test_release_without_ownership_flags_cannot_complete_installation(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + package = directory / "node_modules/@diffexai/diffex" + package.mkdir(parents=True) + tarball = f"{PINS['npm_registry']}@diffexai/diffex/-/diffex-{PINS['diffex_version']}.tgz" + installed = {"version": PINS["diffex_version"], "integrity": PINS["diffex_integrity"], "resolved": tarball} + (package / "package.json").write_text(json.dumps({"version": PINS["diffex_version"]})) + (directory / "package-lock.json").write_text(json.dumps({"packages": {"node_modules/@diffexai/diffex": installed}})) + (directory / "npm-metadata.json").write_text(json.dumps({"version": PINS["diffex_version"], + "dist": {"integrity": PINS["diffex_integrity"], "tarball": tarball}})) + (directory / "version.stdout.txt").write_text(PINS["diffex_version"]) + source = files("diffex_terminal_bench").joinpath("verify-install.cjs").read_text() + script = directory / "verify-install.cjs" + script.write_text(source.replace('"/opt/diffex-terminal-bench/cli"', json.dumps(temp))) + env = {"PATH": os.environ["PATH"], "SETUP_LOGS": temp, "NPM_REGISTRY": PINS["npm_registry"], + "DIFFEX_VERSION": PINS["diffex_version"], "DIFFEX_INTEGRITY": PINS["diffex_integrity"], + "SETUP_ELAPSED_SEC": "1", "SETUP_TIMEOUT_SEC": "600"} + for help_text, valid in (("--help", False), ("--background-process-lifetime", False), + ("--background-process-lifetime --background-process-ownership-dir", True)): + (directory / "help.stdout.txt").write_text(help_text) + result = subprocess.run(["node", str(script), "complete"], env=env, capture_output=True, text=True, timeout=10) + with self.subTest(help_text=help_text): + self.assertEqual(result.returncode == 0, valid, result.stderr) + self.assertEqual((directory / "installation.json").exists(), valid) + if not valid: + self.assertIn("publish and pin the corrected release", result.stderr) + def test_registry_metadata_must_match_all_pins(self): tarball = f"{PINS['npm_registry']}@diffexai/diffex/-/diffex-{PINS['diffex_version']}.tgz" cases = ( diff --git a/evals/terminal-bench/tests/test_outcome.py b/evals/terminal-bench/tests/test_outcome.py index 65b949f76..be3c5667a 100644 --- a/evals/terminal-bench/tests/test_outcome.py +++ b/evals/terminal-bench/tests/test_outcome.py @@ -7,7 +7,7 @@ from harbor.agents.installed.base import AgentAuthenticationError -from diffex_terminal_bench.outcome import DiffexExecutionError, parse_outcome, require_completed +from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError, parse_outcome, require_completed def attempt(reason="stop", error="", retry=False): @@ -17,20 +17,45 @@ def attempt(reason="stop", error="", retry=False): {"type": "agent_end", "messages": [message], "willRetry": retry}] +def write_cleanup(directory): + (directory / "stop-requested").touch() + controls = [{"role": "launcher", "pid": 10, "start": "100"}, {"role": "cli", "pid": 11, "start": "101"}] + for control in controls: + (directory / f"{control['role']}.json").write_text(json.dumps({key: value for key, value in control.items() if key != "role"})) + report = {"schema_version": 1, "phase": "pre-verifier", "stop_requested": True, + "agent_execution_stopped": True, "ownership_validated": True, + "remaining_agent_pids": [], "control_processes": controls, "retained_processes": [], + "ownership_records": []} + (directory / "cleanup.json").write_text(json.dumps(report)) + return report + + def write_evidence(directory, events=None): directory.mkdir(parents=True, exist_ok=True) - config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture"} + config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture", "background_process_lifetime": "environment"} for name, content in { "requested.json": config, "effective.json": {**config, "release_agents_loaded": True, "instruction_delivered": True}, "process.json": {"exit_code": 0, "signal": None, "spawn_error": None}, - "cleanup.json": {"confirmed": True}, }.items(): (directory / name).write_text(json.dumps(content)) + write_cleanup(directory) (directory / "stdout.jsonl").write_text("".join(json.dumps(event) + "\n" for event in [{"type": "session"}, *(events if events is not None else attempt()), {"type": "agent_settled"}])) +def retain_service(directory): + report = write_cleanup(directory) + member = {"pid": 123, "processGroupId": 122, "startTime": "boot:456"} + record = {"version": 1, "executionId": "service", "owner": "environment", "state": "background", + "processGroupId": 122, "supervisorPid": 120, "members": [member]} + (directory / "ownership").mkdir(exist_ok=True) + (directory / "ownership/service.json").write_text(json.dumps(record)) + report.update(ownership_records=[record], retained_processes=[{**member, "executionId": "service", "owner": "environment"}]) + (directory / "cleanup.json").write_text(json.dumps(report)) + return report + + class OutcomeTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -110,6 +135,36 @@ def test_interruption_preserves_partial_evidence(self): self.assertFalse(outcome["stream_complete"]) self.assertTrue((self.directory / "stdout.jsonl").is_file()) + def test_retained_service_does_not_change_execution_outcome(self): + for interruption, code, status in ((None, 0, "completed"), (None, 1, "process_failure"), + ("cancelled", None, "cancelled"), ("timeout", None, "timeout")): + write_evidence(self.directory) + report = retain_service(self.directory) + outcome = parse_outcome(self.directory, interruption=interruption, launcher_exit_code=code) + self.assertEqual(outcome["status"], status) + self.assertTrue(outcome["agent_execution_stopped"]) + self.assertEqual(outcome["retained_processes"], report["retained_processes"]) + + def test_missing_or_contradictory_evidence_blocks_verification(self): + for key, value in (("agent_execution_stopped", False), ("ownership_validated", False), + ("remaining_agent_pids", [124]), ("ownership_records", []), + ("control_processes", []), + ("control_processes", [{"role": "cli", "pid": 123, "start": "456"}])): + write_evidence(self.directory) + report = retain_service(self.directory) + report[key] = value + (self.directory / "cleanup.json").write_text(json.dumps(report)) + for interruption in (None, "cancelled"): + with self.subTest(key=key, interruption=interruption): + outcome = parse_outcome(self.directory, interruption, launcher_exit_code=0) + self.assertFalse(outcome["agent_execution_stopped"]) + with self.assertRaises(DiffexControlStopError): + require_completed(outcome) + write_evidence(self.directory) + retain_service(self.directory) + (self.directory / "ownership/service.json").unlink() + self.assertEqual(parse_outcome(self.directory, launcher_exit_code=0)["status"], "cleanup_failure") + if __name__ == "__main__": unittest.main() diff --git a/evals/terminal-bench/tests/test_parent_exhaustion.py b/evals/terminal-bench/tests/test_parent_exhaustion.py index 3c2d9ec28..45faa6d58 100644 --- a/evals/terminal-bench/tests/test_parent_exhaustion.py +++ b/evals/terminal-bench/tests/test_parent_exhaustion.py @@ -11,6 +11,7 @@ from diffex_terminal_bench.dataset import load_config from diffex_terminal_bench.gate import read_json, sample_tasks, write_json from diffex_terminal_bench.outcome import parse_outcome, require_completed +from test_outcome import write_cleanup class ParentExhaustionTests(unittest.IsolatedAsyncioTestCase): @@ -51,9 +52,9 @@ async def run(): (directory / "stdout.jsonl").write_bytes(stdout) (directory / "stderr.log").write_bytes(stderr) write_json(directory / "process.json", {"exit_code": process.returncode}) - write_json(directory / "cleanup.json", {"confirmed": True}) + write_cleanup(directory) requested = {"provider": "faux", "model": "fixture", "thinking": "off", - "instruction_sha256": "fixture", "tools": []} + "instruction_sha256": "fixture", "tools": [], "background_process_lifetime": "environment"} write_json(directory / "requested.json", requested) write_json(directory / "effective.json", {**requested, "release_agents_loaded": True, "instruction_delivered": True}) diff --git a/evals/terminal-bench/tests/test_recovery.py b/evals/terminal-bench/tests/test_recovery.py index be7cab1c1..0d8563dd5 100644 --- a/evals/terminal-bench/tests/test_recovery.py +++ b/evals/terminal-bench/tests/test_recovery.py @@ -5,14 +5,15 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import create_autospec +from unittest.mock import Mock, create_autospec from unittest.mock import patch from harbor.environments.base import BaseEnvironment, ExecResult from harbor.models.agent.context import AgentContext +from harbor.trial.single_step import SingleStepTrial from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.outcome import DiffexExecutionError +from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError from test_outcome import write_evidence from test_trajectory import assistant, entry @@ -135,18 +136,41 @@ async def execute(**kwargs): return ExecResult(return_code=1 if "cleanup.cjs" in kwargs["command"] else 0, stdout="", stderr="") self.environment.exec.side_effect = execute - with self.assertRaises(DiffexExecutionError): + with self.assertRaises(DiffexControlStopError): await self.agent.run("Task", self.environment, self.context) self.environment.download_dir.assert_awaited_once() self.assertEqual(self.context.metadata["diffex_outcome"]["status"], "infrastructure_failure") async def test_log_collection_failure_is_recorded_locally(self): self.environment.download_dir.side_effect = ConnectionError("fixture") - with self.assertRaises(DiffexExecutionError): + with self.assertRaises(DiffexControlStopError): await self.agent.run("Task", self.environment, self.context) record = json.loads(next(self.logs.glob("run-*/outcome.json")).read_text()) self.assertEqual(record["recovery_errors"][0]["phase"], "log_collection") + async def test_deadline_with_failed_cleanup_blocks_real_harbor_verifier_path(self): + async def execute(**kwargs): + if "run.cjs" in kwargs["command"]: + await asyncio.Event().wait() + return ExecResult(return_code=1 if "cleanup.cjs" in kwargs["command"] else 0, stdout="", stderr="") + + self.environment.exec.side_effect = execute + trial = create_autospec(SingleStepTrial, instance=True) + trial._agent_timeout_sec = 1 + trial._load_trajectory = None + trial.result = self.context + trial.task = Mock() + + async def agent_phase(**kwargs): + await asyncio.wait_for(self.agent.run("Task", self.environment, self.context), 0.05) + + trial._run_agent_phase.side_effect = agent_phase + with self.assertRaises(DiffexControlStopError): + await SingleStepTrial._run_agent(trial) + trial._record_exception.assert_not_called() + trial._sync_agent_output.assert_awaited_once() + self.assertEqual(self.context.metadata["diffex_outcome"]["status"], "cancelled") + if __name__ == "__main__": unittest.main() diff --git a/evals/terminal-bench/tests/test_sampling.py b/evals/terminal-bench/tests/test_sampling.py index a97479936..a68717235 100644 --- a/evals/terminal-bench/tests/test_sampling.py +++ b/evals/terminal-bench/tests/test_sampling.py @@ -37,7 +37,7 @@ async def create(config): self.assertEqual(len(gate["identity"]["sample"]["tasks"]), self.sample_count) self.assertEqual(config.agent.model_name, self.config.model) self.assertEqual(config.agent.kwargs, self.config.policy.model_dump( - include={"thinking", "tools", "subagents", "project_resources"})) + include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"})) self.assertEqual(config.timeout_multiplier, 1) self.assertTrue(config.environment.delete) self.started.append(config.task.path.name)