From 326f1504179410f649a3ec6e7f67e0164913c15f Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 24 Sep 2026 10:17:57 -0400 Subject: [PATCH 1/3] feat: append eval anti-cheating instruction --- .../terminal-bench/scripts/smoke_execution.py | 9 +++++++-- .../src/diffex_terminal_bench/adapter.py | 3 ++- .../src/diffex_terminal_bench/execution.py | 1 + .../src/diffex_terminal_bench/observe.mjs | 4 +++- .../src/diffex_terminal_bench/outcome.py | 4 +++- .../src/diffex_terminal_bench/run.cjs | 1 + evals/terminal-bench/tests/cleanup.test.cjs | 3 ++- evals/terminal-bench/tests/launcher.test.cjs | 3 +++ evals/terminal-bench/tests/test_execution.py | 3 +++ evals/terminal-bench/tests/test_gate.py | 4 ++-- evals/terminal-bench/tests/test_outcome.py | 19 ++++++++++++++++++- .../tests/test_parent_exhaustion.py | 4 +++- 12 files changed, 48 insertions(+), 10 deletions(-) diff --git a/evals/terminal-bench/scripts/smoke_execution.py b/evals/terminal-bench/scripts/smoke_execution.py index a64e9c7ef..708b6e622 100644 --- a/evals/terminal-bench/scripts/smoke_execution.py +++ b/evals/terminal-bench/scripts/smoke_execution.py @@ -19,6 +19,7 @@ from diffex_terminal_bench.adapter import DiffexAgent from diffex_terminal_bench.config import PINS +from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX async def smoke(platform: str) -> None: @@ -91,6 +92,7 @@ async def smoke(platform: str) -> None: assert effective["cwd"] == "/task/work" assert effective["model"] == "gpt-5" and effective["thinking"] == "high" prompt = (run_dir / "system-prompt.txt").read_text() + assert USER_MESSAGE_SUFFIX.strip() not in prompt assert "PRESERVE_TASK_CONTEXT" in prompt and "DO_NOT_LOAD_PROJECT_SYSTEM" not in prompt assert effective["release_agents_sha256"] == PINS["release_agents_sha256"] initial = json.loads((run_dir / "fixture-start.json").read_text()) @@ -130,9 +132,12 @@ async def smoke(platform: str) -> None: assert all(entry["body"]["reasoning_effort"] == "high" for entry in requests) for run_dir in run_dirs: remote_instruction = f"/logs/agent/{run_dir.name}/instruction.txt" - expected_prompt = f'\n{instruction}\n\n' - assert any(message.get("content") in (expected_prompt, [{"type": "text", "text": expected_prompt}]) + expected_prompt = f'\n{instruction}\n\n{USER_MESSAGE_SUFFIX}' + assert any(message.get("role") == "user" and message.get("content") in (expected_prompt, [{"type": "text", "text": expected_prompt}]) for entry in requests for message in entry["body"]["messages"]) + assert not any(USER_MESSAGE_SUFFIX.strip() in str(message.get("content", "")) + for entry in requests for message in entry["body"]["messages"] + if message.get("role") in ("system", "developer")) for run_dir in run_dirs: assert "STALE_HARNESS_SELECTION" not in (run_dir / "system-prompt.txt").read_text() for path in trial.agent_dir.rglob("*"): diff --git a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py index 75adabca5..a13326d4b 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py @@ -16,7 +16,7 @@ from harbor.models.agent.context import AgentContext from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SUBAGENT_TOOLS +from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SUBAGENT_TOOLS, USER_MESSAGE_SUFFIX from diffex_terminal_bench.lifecycle import read_cleanup from diffex_terminal_bench.outcome import DiffexControlStopError, parse_outcome, require_completed from diffex_terminal_bench.transport import retry_transport @@ -112,6 +112,7 @@ async def run( "evolution": {"enabled": False}, "state_policy": "fresh-per-run", "credential_env_names": sorted(credentials), "instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(), + "user_message_suffix": USER_MESSAGE_SUFFIX, } await self.exec_as_root(environment, command=f"umask 077 && mkdir -p {shlex.quote(str(self.environment_logs_dir))} && mkdir {shlex.quote(str(run_dir))}") with tempfile.TemporaryDirectory(prefix="diffex-trial-") as temp: diff --git a/evals/terminal-bench/src/diffex_terminal_bench/execution.py b/evals/terminal-bench/src/diffex_terminal_bench/execution.py index 057c5b77e..0b3a64d3a 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/execution.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/execution.py @@ -18,6 +18,7 @@ } BUILTIN_TOOLS = ("read", "bash", "edit", "write", "update_plan", "grep", "find", "ls") SUBAGENT_TOOLS = ("spawn_agent", "agent_action", "agent_status") +USER_MESSAGE_SUFFIX = "\nDo not cheat by using online solutions or hints specific to this task." class ExecutionOptions(InstalledAgentOptions): diff --git a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs index cb2257694..30bd88114 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs @@ -13,6 +13,7 @@ export default function observe(api) { recorded = true; const instructionPath = join(runDir, "instruction.txt"); const instruction = readFileSync(instructionPath, "utf8"); + const requested = JSON.parse(readFileSync(join(runDir, "requested.json"), "utf8")); const prompt = typeof event.message.content === "string" ? event.message.content : event.message.content.filter((block) => block.type === "text").map((block) => block.text).join(""); const systemPrompt = ctx.getSystemPrompt(); @@ -26,7 +27,8 @@ export default function observe(api) { background_process_lifetime: process.argv.includes("--background-process-lifetime") ? process.argv[process.argv.indexOf("--background-process-lifetime") + 1] : null, instruction_transport: "native-file", - instruction_delivered: prompt === `\n${instruction}\n\n`, + instruction_delivered: prompt === `\n${instruction}\n\n${requested.user_message_suffix}`, + user_message_suffix: requested.user_message_suffix, instruction_sha256: createHash("sha256").update(instruction).digest("hex"), prompt_sha256: createHash("sha256").update(prompt).digest("hex"), system_prompt_sha256: createHash("sha256").update(systemPrompt).digest("hex"), diff --git a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py index a4ccae89e..56f4da02b 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py @@ -7,6 +7,7 @@ from harbor.agents.installed.base import AgentAuthenticationError, NonZeroAgentExitCodeError from diffex_terminal_bench.lifecycle import read_cleanup +from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX class DiffexExecutionError(NonZeroAgentExitCodeError): @@ -140,7 +141,8 @@ def parse_outcome(directory: Path, interruption: str | None = None, effective = json.loads((directory / "effective.json").read_text()) if not isinstance(requested, dict) or not isinstance(effective, dict): raise ValueError("invalid_configuration") - matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime")) + matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime", "user_message_suffix")) + matches = matches and requested["user_message_suffix"] == USER_MESSAGE_SUFFIX matches = matches and requested.get("background_process_lifetime") == "environment" matches = matches and effective.get("release_agents_loaded") is True and effective.get("instruction_delivered") is True matches = matches and sorted(effective["tools"]) == sorted(requested["tools"]) diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs index 3c4c0e412..3494fff43 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs @@ -45,6 +45,7 @@ const args = [ "--background-process-ownership-dir", join(runDir, "ownership"), "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--extension", join(runDir, "observe.mjs"), "--print", `@${join(runDir, "instruction.txt")}`, + config.user_message_suffix, ]; const stdout = openSync(join(runDir, "stdout.jsonl"), "wx", 0o600); const stderr = openSync(join(runDir, "stderr.log"), "wx", 0o600); diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs index 6d624a8aa..05fbe8ba7 100644 --- a/evals/terminal-bench/tests/cleanup.test.cjs +++ b/evals/terminal-bench/tests/cleanup.test.cjs @@ -154,7 +154,8 @@ if (process.argv[2] === "service") { test("launcher death before PID publication cannot release the CLI gate", options, async (t) => { const f = await fixture(t); writeFileSync(join(f.runDir, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", - thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment" })); + thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment", + user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task." })); writeFileSync(join(f.runDir, "release-agents.md"), "Fixture context"); const launcher = spawn(process.execPath, ["--require", join(__dirname, "fixtures/kill-before-cli-publication.cjs"), join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir], { diff --git a/evals/terminal-bench/tests/launcher.test.cjs b/evals/terminal-bench/tests/launcher.test.cjs index fd1ab719d..e4451df8a 100644 --- a/evals/terminal-bench/tests/launcher.test.cjs +++ b/evals/terminal-bench/tests/launcher.test.cjs @@ -17,6 +17,7 @@ test("launcher preserves task packages outside exported logs and isolates each r writeFileSync(join(logs, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", thinking: "high", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment", + user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task.", })); writeFileSync(join(logs, "release-agents.md"), "fixture context"); const child = new EventEmitter(); @@ -32,6 +33,8 @@ test("launcher preserves task packages outside exported logs and isolates each r childEnv = options.env; assert.equal(command, "/bin/sh"); assert.ok(args.includes(join(logs, "ownership"))); + assert.equal(args.at(-2), `@${join(logs, "instruction.txt")}`); + assert.equal(args.at(-1), "\nDo not cheat by using online solutions or hints specific to this task."); return child; } }; return require(name); diff --git a/evals/terminal-bench/tests/test_execution.py b/evals/terminal-bench/tests/test_execution.py index 6118cda79..59b5f20ec 100644 --- a/evals/terminal-bench/tests/test_execution.py +++ b/evals/terminal-bench/tests/test_execution.py @@ -18,6 +18,7 @@ from diffex_terminal_bench.benchmark import PACKAGE, run_identity, trial_config from diffex_terminal_bench.config import PINS from diffex_terminal_bench.dataset import load_config +from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX class ExecutionTests(unittest.IsolatedAsyncioTestCase): @@ -87,6 +88,8 @@ async def upload(source, target): self.assertNotIn("spawn_agent", configs[0]["tools"]) self.assertEqual(configs[0]["evolution"], {"enabled": False}) self.assertEqual(configs[0]["background_process_lifetime"], "environment") + self.assertEqual(configs[0]["user_message_suffix"], USER_MESSAGE_SUFFIX) + self.assertEqual(USER_MESSAGE_SUFFIX, "\nDo not cheat by using online solutions or hints specific to this task.") calls = [call.kwargs for call in environment.exec.call_args_list if call.kwargs.get("env")] self.assertEqual(len(calls), 2) for call in calls: diff --git a/evals/terminal-bench/tests/test_gate.py b/evals/terminal-bench/tests/test_gate.py index 0d2eeb29a..219a6b463 100644 --- a/evals/terminal-bench/tests/test_gate.py +++ b/evals/terminal-bench/tests/test_gate.py @@ -17,7 +17,7 @@ from diffex_terminal_bench.benchmark import PACKAGE, main, run_stage from diffex_terminal_bench.config import PINS from diffex_terminal_bench.dataset import BenchmarkConfig, load_config -from diffex_terminal_bench.execution import SUBAGENT_TOOLS +from diffex_terminal_bench.execution import SUBAGENT_TOOLS, USER_MESSAGE_SUFFIX from diffex_terminal_bench.gate import inventory, require_previous, sha256, stage_tasks, validate_trial, write_json from diffex_terminal_bench.outcome import parse_outcome from diffex_terminal_bench.trajectory import export_trajectory @@ -38,7 +38,7 @@ def setUp(self): "release_agents_sha256": "release-hash", "task_digests": {"task": "sha256:task"}} requested = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "subagents": "disabled", "project_resources": "context-only", "background_process_lifetime": "environment", "pins": PINS, - "evolution": {"enabled": False}, "state_policy": "fresh-per-run"} + "evolution": {"enabled": False}, "state_policy": "fresh-per-run", "user_message_suffix": USER_MESSAGE_SUFFIX} (self.run / "instruction.txt").write_text("task") (self.run / "system-prompt.txt").write_text("system") requested["instruction_sha256"] = sha256(self.run / "instruction.txt") diff --git a/evals/terminal-bench/tests/test_outcome.py b/evals/terminal-bench/tests/test_outcome.py index be3c5667a..92ef70706 100644 --- a/evals/terminal-bench/tests/test_outcome.py +++ b/evals/terminal-bench/tests/test_outcome.py @@ -7,6 +7,7 @@ from harbor.agents.installed.base import AgentAuthenticationError +from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX from diffex_terminal_bench.outcome import DiffexControlStopError, DiffexExecutionError, parse_outcome, require_completed @@ -32,7 +33,8 @@ def write_cleanup(directory): def write_evidence(directory, events=None): directory.mkdir(parents=True, exist_ok=True) - config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture", "background_process_lifetime": "environment"} + config = {"provider": "openai", "model": "gpt-5", "thinking": "high", "tools": ["bash"], "instruction_sha256": "fixture", "background_process_lifetime": "environment", + "user_message_suffix": USER_MESSAGE_SUFFIX} for name, content in { "requested.json": config, "effective.json": {**config, "release_agents_loaded": True, "instruction_delivered": True}, @@ -135,6 +137,21 @@ def test_interruption_preserves_partial_evidence(self): self.assertFalse(outcome["stream_complete"]) self.assertTrue((self.directory / "stdout.jsonl").is_file()) + def test_user_message_suffix_must_be_delivered_and_match_the_eval_policy(self): + for field, value in (("user_message_suffix", ""), ("instruction_delivered", False)): + write_evidence(self.directory) + path = self.directory / "effective.json" + effective = json.loads(path.read_text()) + effective[field] = value + path.write_text(json.dumps(effective)) + self.assertEqual(parse_outcome(self.directory, launcher_exit_code=0)["status"], "configuration_failure") + for name in ("requested.json", "effective.json"): + path = self.directory / name + data = json.loads(path.read_text()) + data.update(user_message_suffix="arbitrary task hint", instruction_delivered=True) + path.write_text(json.dumps(data)) + self.assertEqual(parse_outcome(self.directory, launcher_exit_code=0)["status"], "configuration_failure") + def test_retained_service_does_not_change_execution_outcome(self): for interruption, code, status in ((None, 0, "completed"), (None, 1, "process_failure"), ("cancelled", None, "cancelled"), ("timeout", None, "timeout")): diff --git a/evals/terminal-bench/tests/test_parent_exhaustion.py b/evals/terminal-bench/tests/test_parent_exhaustion.py index 45faa6d58..21b61a403 100644 --- a/evals/terminal-bench/tests/test_parent_exhaustion.py +++ b/evals/terminal-bench/tests/test_parent_exhaustion.py @@ -9,6 +9,7 @@ from diffex_terminal_bench.benchmark import PACKAGE, REPOSITORY, StageFailedError, run_stage from diffex_terminal_bench.dataset import load_config +from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX from diffex_terminal_bench.gate import read_json, sample_tasks, write_json from diffex_terminal_bench.outcome import parse_outcome, require_completed from test_outcome import write_cleanup @@ -54,7 +55,8 @@ async def run(): write_json(directory / "process.json", {"exit_code": process.returncode}) write_cleanup(directory) requested = {"provider": "faux", "model": "fixture", "thinking": "off", - "instruction_sha256": "fixture", "tools": [], "background_process_lifetime": "environment"} + "instruction_sha256": "fixture", "tools": [], "background_process_lifetime": "environment", + "user_message_suffix": USER_MESSAGE_SUFFIX} write_json(directory / "requested.json", requested) write_json(directory / "effective.json", {**requested, "release_agents_loaded": True, "instruction_delivered": True}) From ac04f1c2bac17bad69104d474e347ded2c031dd4 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Thu, 24 Sep 2026 10:26:39 -0400 Subject: [PATCH 2/3] feat(agent): fix terminal-bench eval prompts --- evals/terminal-bench/README.md | 46 ++++++++++--------- evals/terminal-bench/RELEASE_GATE.md | 10 ++-- evals/terminal-bench/execution.yaml | 2 +- evals/terminal-bench/full.yaml | 3 +- .../terminal-bench/scripts/smoke_execution.py | 35 ++++++++------ .../src/diffex_terminal_bench/adapter.py | 7 ++- .../src/diffex_terminal_bench/benchmark.py | 4 +- .../eval-system-prompt.txt | 39 ++++++++++++++++ .../src/diffex_terminal_bench/execution.py | 7 ++- .../src/diffex_terminal_bench/gate.py | 4 +- .../src/diffex_terminal_bench/observe.mjs | 6 +-- .../src/diffex_terminal_bench/outcome.py | 7 +-- .../src/diffex_terminal_bench/run.cjs | 12 +---- evals/terminal-bench/tests/cleanup.test.cjs | 1 - evals/terminal-bench/tests/launcher.test.cjs | 3 +- evals/terminal-bench/tests/test_execution.py | 26 +++++++---- evals/terminal-bench/tests/test_gate.py | 19 ++++---- evals/terminal-bench/tests/test_outcome.py | 23 ++++++++-- .../tests/test_parent_exhaustion.py | 6 +-- 19 files changed, 166 insertions(+), 94 deletions(-) create mode 100644 evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt diff --git a/evals/terminal-bench/README.md b/evals/terminal-bench/README.md index e5abe3e91..94b97466d 100644 --- a/evals/terminal-bench/README.md +++ b/evals/terminal-bench/README.md @@ -121,12 +121,13 @@ The unit checks cover installation pins, configuration validation, explicit cred They also cover outcome parsing, incomplete streams, transport failures, log-collection failures, and repeated cancellation during recovery. Converter checks cover deduplication, tool observations, nested child trajectories, auxiliary usage, unknown telemetry, images, and partial messages. The execution smoke installs the real npm release, then redirects a fixture model to a loopback faux provider with a test-only wrapper. -It exercises shell and file tools, multiline instructions with shell metacharacters and Unicode, repeated trials, enabled child agents, and non-root execution without paid API calls. +It exercises shell and file tools, multiline instructions with shell metacharacters and Unicode, repeated trials, and non-root execution without paid API calls. The fixture checks the actual request credentials without storing their values, verifies empty initial sessions and state, and scans retained artifacts for the fixture secrets. -It also checks that task context and the exact release instructions reach the system prompt while project settings, extensions, and `SYSTEM.md` are ignored. +It checks the supplied eval system prompt and the anti-cheating sentence in the user message against actual provider requests. +Automatic project context, packaged release instructions, project settings, extensions, and `SYSTEM.md` are excluded from the system prompt. Only the agent log directory is mounted, and each invocation removes its container in a `finally` block. Generated evidence is retained under `results/diffex-execution-*/`. -Each execution smoke validates ATIF with Harbor's validator and reconciles nonzero input, output, cached-token, and cost totals against every root and child response. +Each execution smoke validates ATIF with Harbor's validator and reconciles nonzero input, output, cached-token, and cost totals against every response. The outcome smoke exercises success, authentication failure, exhausted retry, recovered retry, interrupted JSON output, and a Harbor-style execution timeout as both root and a non-root user. It uses the pinned npm release with a loopback provider and checks nonzero terminal-error exits and rejection of truncated output even when the process exits zero. Success, provider-failure, forced-CLI-death, and timeout cases create a TERM-resistant background service and verify that its PID and start time are unchanged across adapter cleanup. @@ -256,7 +257,8 @@ uv run --locked harbor run --config execution.yaml \ [execution.yaml](execution.yaml) records the default policy and allows 630 seconds for Harbor setup, encompassing the installer's 600-second deadline and transport overhead. The task's agent timeout remains in effect. -Use `--ak thinking=medium`, `--ak tools=read,bash,edit,write`, or `--ak subagents=disabled` to change the corresponding policy explicitly. +Use `--ak thinking=medium` or `--ak tools=read,bash,edit,write` to change the corresponding policy explicitly. +Subagents must remain disabled; `--ak subagents=enabled` is rejected during preflight. Inspect accepted options with `uv run --locked harbor agent schema diffex_terminal_bench.adapter:DiffexAgent`. Unknown options, arbitrary native config, prompt templates, injected skills, and MCP servers are rejected. Model IDs must include a supported provider prefix; fuzzy patterns and thinking suffixes are rejected. @@ -287,8 +289,10 @@ The adapter uploads the complete UTF-8 instruction and runs `diffex --mode json The CLI receives stdin EOF through `/dev/null` and the launcher's ignored stdin. Native `@file` transport adds Diffex's standard `` wrapper while retaining all instruction bytes, including leading/trailing whitespace and CRLF line endings. Instruction contents never become shell command text or command-line flags. -The observer extension checks that the first user message contains exactly that wrapper and the original instruction. -It registers only a `message_start` observer, preserving compatibility with child agents. +The launcher appends `Do not cheat by using online solutions or hints specific to this task.` after the native file wrapper, separated by a blank line, in the same user message. +The original instruction file and its hash remain unchanged. +The observer extension checks that the first user message contains exactly the wrapper, original instruction, and anti-cheating suffix. +It registers only a `message_start` observer and does not rewrite provider requests. Each invocation creates an exclusive `run-/` under Harbor's agent logs. Its `DIFFEX_CODING_AGENT_DIR` is a fresh private `/tmp/diffex-tb-state-*` directory on sandbox disk, with a fresh home and global `settings.json` containing `{"evolution":{"enabled":false}}` before startup. @@ -296,28 +300,28 @@ Packages and caches remain outside the exported logs and survive until sandbox t No prior settings, credentials, sessions, or evolved harness selections are copied into it. Startup network updates are disabled with the supported offline setting so the pinned model catalog remains stable. -The `context-only` project policy retains ancestor `AGENTS.md`/`CLAUDE.md` context and disables project trust, discovered extensions, skills, prompt templates, and themes. -It does not pass `--no-context-files` or replace the system prompt. -The pinned npm `0.3.1` release includes `dist/AGENTS.md`, which matches the adapter's checked-in release instructions. -The adapter retains that packaged context without adding a global duplicate. -The effective system prompt must contain the release instructions, and their SHA-256 is recorded. +The eval adapter passes `--system-prompt` with the checked-in [eval prompt](src/diffex_terminal_bench/eval-system-prompt.txt) and `--no-context-files`. +The entire automatic `` section is omitted, including packaged release instructions and ancestor `AGENTS.md`/`CLAUDE.md` files and their paths. +Task files remain on disk and can be read explicitly by the agent. +The `context-only` project resource setting continues to disable project trust, discovered extensions, skills, prompt templates, and themes; eval launches additionally suppress automatic context files. +The effective system prompt must match the checked-in prompt's SHA-256, and execution evidence must confirm context files are disabled. +This uses supported flags in the pinned Diffex release and requires no product code or release changes. -The checked-in `benchmark.toml` disables subagents, omitting `spawn_agent`, `agent_action`, and `agent_status` from the explicit tool allowlist. -The separate controlled smoke configuration in `execution.yaml` enables subagents and includes those tools. -Diffex decides whether to call those tools; enabling them does not require a child on every task. -Children share their trial's sandbox, and the pinned CLI owns child scheduling and its default concurrency limit. -The staged runner's `-n/--concurrency` controls simultaneous trials and sandboxes separately. -An adapter invocation that omits the subagent policy still defaults to disabled, omitting coordination tools. -Root and child sessions are retained together beneath `run-/sessions/`, including the parent-owned child session directories. +All eval configurations require disabled subagents and omit `spawn_agent`, `agent_action`, and `agent_status` from the tool allowlist. +The pinned Diffex 0.3.1 CLI does not propagate `--system-prompt` or `--no-context-files` to child agents, so enabled subagents cannot satisfy this prompt policy. +The adapter rejects attempts to enable them before setup or execution. +The staged runner's `-n/--concurrency` still controls simultaneous independent trials and sandboxes. +Native sessions remain beneath `run-/sessions/`; trajectory conversion continues to support retained child sessions from earlier evaluations. ### Execution evidence Each run directory retains: -- `requested.json`: pins, model, thinking, tools, subagent policy, project policy, credential variable names, and instruction hash. +- `requested.json`: pins, model, thinking, tools, subagent policy, project policy, credential variable names, instruction hash, user-message suffix, and system prompt hash. - `instruction.txt`: the original instruction. -- `effective.json`: resolved model, thinking, tools, working directory, session path, prompt hashes, and release-context verification. -- `system-prompt.txt`: the effective system prompt, including release and task context. +- `eval-system-prompt.txt`: the requested eval system prompt. +- `effective.json`: resolved model, thinking, tools, working directory, session path, prompt hashes, user-message verification, and context-file suppression. +- `system-prompt.txt`: the observed system prompt, without automatic project context. - `stdout.jsonl`, `stderr.log`, and `process.json`: native JSON events, diagnostics, and process status. - `launcher.json` and `cli.json`: launcher and root CLI PID/start-time identities. - `ownership/*.json`: durable shell ownership records published by the CLI. diff --git a/evals/terminal-bench/RELEASE_GATE.md b/evals/terminal-bench/RELEASE_GATE.md index fd2854058..bea0fc043 100644 --- a/evals/terminal-bench/RELEASE_GATE.md +++ b/evals/terminal-bench/RELEASE_GATE.md @@ -29,8 +29,8 @@ The preparation command writes `results/release-gate/dataset.json` without creat Harbor 0.23.0 and Daytona SDK 0.210.0 are locked in the benchmark's private Python environment. Node 22.23.2, Diffex 0.3.1, npm integrity, registry, and Node archive checksums come from [pins.toml](src/diffex_terminal_bench/pins.toml). The default is `openai/gpt-5.6-luna`, `max` thinking, tools `read,bash,edit,write,update_plan`, disabled subagents, disabled evolution, and `context-only` project resources. -Enabled subagents add `spawn_agent`, `agent_action`, and `agent_status` to the model's tools; Diffex chooses whether to use them. -Children execute inside the parent trial's sandbox and their sessions and usage remain in the trial evidence. +Enabled subagents are rejected because the pinned CLI does not propagate eval prompt and context-file flags to children. +Every new eval uses the checked-in system prompt without automatic project context and appends the anti-cheating instruction to the user message. Change and commit the configuration before starting a new qualification sequence. Every subsequent stage must match the exact recorded identity, including the adapter commit and dependency lock. The earlier TB 2.0 / Diffex 0.2.2 oracle evidence cannot qualify this configuration. @@ -237,10 +237,10 @@ Use `-n 1` throughout that sequence while capacity remains constrained; task dea Each stage writes `gate.json` before starting remote work and updates it after every completed trial, including failures. The report separates verified `trials` from `failures`, records `completed_trials` and `expected_trials`, and keeps `failure` as the first observed failure. Failure entries identify the task, trial index, evidence directory, error type, and phase; a cleanup error following an execution error is retained in `additional_errors`. -It records dataset/archive and per-task digests, adapter commit, Harbor/Daytona/Python versions, dependency lock hash, installation pins, model, thinking, tools, subagent policy, and hashes of adapter/harness files and the release instructions. +It records dataset/archive and per-task digests, adapter commit, Harbor/Daytona/Python versions, dependency lock hash, installation pins, model, thinking, tools, subagent policy, and hashes of adapter/harness files and the eval system prompt. Each Diffex trial additionally retains the effective system prompt and its SHA-256. -Full system prompt hashes can differ between isolated trials because context includes trial-specific paths. -The release instructions and harness source identity must remain consistent. +The effective system prompt must exactly match the checked-in eval prompt across isolated trials. +The user-message suffix, disabled automatic context files, and harness source identity are also verified. Trial directories contain Harbor `config.json`, `lock.json`, `result.json`, verifier reward and combined stdout/stderr, agent logs, native sessions, and sandbox lifecycle evidence. Diffex installation evidence includes the three CLI checks and verified npm/Node pins. diff --git a/evals/terminal-bench/execution.yaml b/evals/terminal-bench/execution.yaml index e564b896a..b8bce5479 100644 --- a/evals/terminal-bench/execution.yaml +++ b/evals/terminal-bench/execution.yaml @@ -11,6 +11,6 @@ agents: kwargs: thinking: high tools: read,bash,edit,write,update_plan - subagents: enabled + subagents: disabled project_resources: context-only background_process_lifetime: environment diff --git a/evals/terminal-bench/full.yaml b/evals/terminal-bench/full.yaml index 633cda18d..be6d3d937 100644 --- a/evals/terminal-bench/full.yaml +++ b/evals/terminal-bench/full.yaml @@ -14,7 +14,8 @@ agents: kwargs: thinking: max tools: read,bash,edit,write,update_plan - subagents: enabled + # Child agents in the pinned CLI do not inherit eval prompt flags. + subagents: disabled project_resources: context-only background_process_lifetime: environment env: diff --git a/evals/terminal-bench/scripts/smoke_execution.py b/evals/terminal-bench/scripts/smoke_execution.py index 708b6e622..6f1032662 100644 --- a/evals/terminal-bench/scripts/smoke_execution.py +++ b/evals/terminal-bench/scripts/smoke_execution.py @@ -19,7 +19,7 @@ from diffex_terminal_bench.adapter import DiffexAgent from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX +from diffex_terminal_bench.execution import SYSTEM_PROMPT, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX async def smoke(platform: str) -> None: @@ -57,10 +57,10 @@ async def smoke(platform: str) -> None: await agent.exec_as_root(environment, command="mv /usr/local/bin/diffex /usr/local/bin/diffex-release") await environment.upload_file(wrapper, "/usr/local/bin/diffex") await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex && mkdir -p /task/work/.diffex/extensions") - # Task context must remain visible while executable resources and - # project settings cannot replace the chosen benchmark harness. + # Automatic context and executable project resources must not + # alter the eval prompt, even when the task supplies those files. for name, content in { - "AGENTS.md": "Task context marker: PRESERVE_TASK_CONTEXT\n", + "AGENTS.md": "Task context marker: DO_NOT_LOAD_TASK_CONTEXT\n", ".diffex/settings.json": '{"evolution":{"enabled":true},"defaultThinkingLevel":"off"}', ".diffex/SYSTEM.md": "DO_NOT_LOAD_PROJECT_SYSTEM", ".diffex/extensions/trap.mjs": 'throw new Error("DO_NOT_LOAD_PROJECT_EXTENSION");', @@ -73,11 +73,11 @@ async def smoke(platform: str) -> None: await agent.exec_as_root(environment, command="wget --retry-connrefused --tries=20 --waitretry=1 -qO /dev/null http://127.0.0.1:8765/health") instruction = "\n --literal @file 'single' \"double\" $HOME $(touch /task/work/INJECTED) `pwd`\r\nWrite shell.txt and result.txt. Unicode 雪.\n\n" run_dirs = [] - for subagents, user in (("disabled", None), ("disabled", None), ("enabled", None), ("disabled", "65534")): + for user in (None, None, "65534"): if user is not None: await agent.exec_as_root(environment, command="chown -R 65534 /task/work") environment.default_user = user - current = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", subagents=subagents, + current = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", subagents="disabled", extra_env={"OPENAI_API_KEY": "fixture-secret-never-log"}) before = set(trial.agent_dir.glob("run-*")) context = AgentContext() @@ -93,10 +93,10 @@ async def smoke(platform: str) -> None: assert effective["model"] == "gpt-5" and effective["thinking"] == "high" prompt = (run_dir / "system-prompt.txt").read_text() assert USER_MESSAGE_SUFFIX.strip() not in prompt - assert "PRESERVE_TASK_CONTEXT" in prompt and "DO_NOT_LOAD_PROJECT_SYSTEM" not in prompt - assert effective["release_agents_sha256"] == PINS["release_agents_sha256"] + assert prompt == SYSTEM_PROMPT + assert effective["system_prompt_sha256"] == SYSTEM_PROMPT_SHA256 + assert effective["context_files_disabled"] is True initial = json.loads((run_dir / "fixture-start.json").read_text()) - assert effective["release_agents_path"] == "/opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist/AGENTS.md" assert initial == {"state_entries": ["home", "settings.json"], "session_entries": [], "settings": {"evolution": {"enabled": False}}, "unrelated_credential_present": False} assert (run_dir / "sessions/root.jsonl").stat().st_size > 0 @@ -105,12 +105,11 @@ async def smoke(platform: str) -> None: assert not (run_dir / "state").exists() events = [json.loads(line) for line in (run_dir / "stdout.jsonl").read_text().splitlines()] assert not any(event.get("isError") for event in events) - if subagents == "enabled": - assert len(list((run_dir / "sessions").rglob("*.jsonl"))) >= 2 + assert len(list((run_dir / "sessions").rglob("*.jsonl"))) == 1 validator = TrajectoryValidator() assert validator.validate(trial.agent_dir / "trajectory.json"), validator.get_errors() trajectory = json.loads((trial.agent_dir / "trajectory.json").read_text()) - assert bool(trajectory.get("subagent_trajectories")) == (subagents == "enabled") + assert not trajectory.get("subagent_trajectories") messages = [record["message"] for path in (run_dir / "sessions").rglob("*.jsonl") for line in path.read_text().splitlines() if (record := json.loads(line)).get("type") == "message" and record["message"].get("role") == "assistant"] @@ -126,10 +125,20 @@ async def smoke(platform: str) -> None: await current.exec_as_agent(environment, command=shlex.join([ "/opt/diffex-terminal-bench/node/bin/node", "/tmp/faux-provider.cjs", "contaminate", state["state_dir"], ])) - await agent.exec_as_root(environment, command="test ! -e /task/work/INJECTED && test \"$(cat cwd.txt)\" = /task/work && test \"$(cat shell.txt)\" = shell-ok && test \"$(cat result.txt)\" = file-ok && test \"$(cat child.txt)\" = child-ok") + await agent.exec_as_root(environment, command="test ! -e /task/work/INJECTED && test \"$(cat cwd.txt)\" = /task/work && test \"$(cat shell.txt)\" = shell-ok && test \"$(cat result.txt)\" = file-ok") requests = [json.loads(line) for line in (trial.agent_dir / "fixture-requests.jsonl").read_text().splitlines()] assert requests and all(entry["credential_received"] for entry in requests) assert all(entry["body"]["reasoning_effort"] == "high" for entry in requests) + for entry in requests: + system_messages = [message["content"] for message in entry["body"]["messages"] + if message.get("role") in ("system", "developer")] + assert system_messages + for content in system_messages: + text = content if isinstance(content, str) else "".join(block["text"] for block in content) + assert text == SYSTEM_PROMPT + for excluded in ("", "\n{instruction}\n\n{USER_MESSAGE_SUFFIX}' diff --git a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py index a13326d4b..d7e963db5 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py @@ -16,7 +16,7 @@ from harbor.models.agent.context import AgentContext from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SUBAGENT_TOOLS, USER_MESSAGE_SUFFIX +from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX from diffex_terminal_bench.lifecycle import read_cleanup from diffex_terminal_bench.outcome import DiffexControlStopError, parse_outcome, require_completed from diffex_terminal_bench.transport import retry_transport @@ -101,8 +101,6 @@ async def run( if not isinstance(options, ExecutionOptions): raise TypeError("Expected validated execution options") tools = options.tools.split(",") - if options.subagents == "enabled": - tools.extend(SUBAGENT_TOOLS) run_dir = self.environment_logs_dir / f"run-{uuid.uuid4().hex}" requested = { "pins": PINS, "provider": provider, "model": model, @@ -113,6 +111,7 @@ async def run( "credential_env_names": sorted(credentials), "instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(), "user_message_suffix": USER_MESSAGE_SUFFIX, + "system_prompt_sha256": SYSTEM_PROMPT_SHA256, } await self.exec_as_root(environment, command=f"umask 077 && mkdir -p {shlex.quote(str(self.environment_logs_dir))} && mkdir {shlex.quote(str(run_dir))}") with tempfile.TemporaryDirectory(prefix="diffex-trial-") as temp: @@ -120,7 +119,7 @@ async def run( source = Path(temp, name) source.write_text(content, encoding="utf-8") await environment.upload_file(source, str(run_dir / name)) - for name in ("run.cjs", "cleanup.cjs", "evidence.cjs", "observe.mjs", "release-agents.md"): + for name in ("run.cjs", "cleanup.cjs", "evidence.cjs", "observe.mjs", "eval-system-prompt.txt"): with as_file(files(__package__).joinpath(name)) as source: await environment.upload_file(source, str(run_dir / name)) # Uploads may be root-owned even when the task selects a non-root user. diff --git a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py b/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py index 5356f9f53..1bb48b655 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py @@ -24,7 +24,7 @@ from diffex_terminal_bench.dataset import BenchmarkConfig, load_config, prepare_dataset from diffex_terminal_bench.daytona_backend import confirm_cleanup from diffex_terminal_bench.diagnostics import TrialExecutionFailure, describe_failure, diagnose_trial -from diffex_terminal_bench.execution import PROVIDER_ENV +from diffex_terminal_bench.execution import PROVIDER_ENV, SYSTEM_PROMPT_SHA256 from diffex_terminal_bench.retention import cleanup_trial_volume from diffex_terminal_bench.gate import STAGES, fingerprint, inventory, read_json, require_previous, sample_tasks, sha256, stage_tasks, validate_trial, verify_stage, write_json from diffex_terminal_bench.summary import refresh_index, write_index @@ -78,7 +78,7 @@ def run_identity(config: BenchmarkConfig, digests: dict) -> dict: "daytona_target": os.environ.get("DAYTONA_TARGET", "organization-default"), "daytona_api_url": os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api"), "harness_files": {path.name: sha256(path) for path in sorted(Path(__file__).parent.iterdir()) if path.is_file()}, - "release_agents_sha256": PINS["release_agents_sha256"], + "system_prompt_sha256": SYSTEM_PROMPT_SHA256, "timeout_multiplier": 1.0, "resource_overrides": None, "concurrency": config.concurrency, "harbor_retries": 0} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt b/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt new file mode 100644 index 000000000..a024a16ac --- /dev/null +++ b/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt @@ -0,0 +1,39 @@ +Available tools: +- read: Read file contents +- bash: Execute bash commands (ls, grep, find, etc.) +- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call +- write: Create or overwrite files +- update_plan: track progress with an ordered task checklist + +Tool usage guidelines: +- read: + - Use read to examine files instead of cat or sed. +- bash: + - Use bash for file operations like ls, rg, find + - You can inspect DIFFEX_* environment variables for current model and session details. +- edit: + - Use edit for precise changes (edits[].oldText must match exactly) + - When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls + - Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit. + - Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions. +- write: + - Use write only for new files or complete rewrites. +- update_plan: + - Use update_plan for complex, ambiguous, dependent, long-running, or multi-part work, when the user requests a plan, or when checkpoints would improve collaboration; skip it for simple or immediately executable work. + - Write short, ordered, outcome-focused steps with observable evidence where useful; do not narrate commands or pad the checklist with obvious actions. + - Include final verification when correctness requires it. + - Update the checklist at meaningful milestones and mark completed work before moving on. + - Explain material replanning, omit explanations for routine progress, and use clear: true only when abandoning the checklist. + - Publishing the checklist communicates progress but does not execute, schedule, or prove completion of its steps. + +Conversational Style: +- Keep responses concise and direct, adapting detail and terminology to the user. +- No emojis anywhere. +- No fluff or cheerful filler text (e.g., "Thanks @user" not "Thanks so much @user!"). +- Show file paths clearly when working with files + +Quality: +- Follow repository instructions, conventions, and existing tooling. + +Diffex documentation (read the documentation website only when the user asks about Diffex itself, its SDK, extensions, or skills): +- Documentation website: https://diffex.ai/docs/overview/ diff --git a/evals/terminal-bench/src/diffex_terminal_bench/execution.py b/evals/terminal-bench/src/diffex_terminal_bench/execution.py index 0b3a64d3a..79046e485 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/execution.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/execution.py @@ -1,5 +1,7 @@ """Explicit execution policy for the pinned Diffex release.""" +import hashlib +from importlib.resources import files from typing import Literal from harbor.agents.options import InstalledAgentOptions @@ -19,12 +21,15 @@ BUILTIN_TOOLS = ("read", "bash", "edit", "write", "update_plan", "grep", "find", "ls") SUBAGENT_TOOLS = ("spawn_agent", "agent_action", "agent_status") USER_MESSAGE_SUFFIX = "\nDo not cheat by using online solutions or hints specific to this task." +SYSTEM_PROMPT = files(__package__).joinpath("eval-system-prompt.txt").read_text(encoding="utf-8") +SYSTEM_PROMPT_SHA256 = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest() class ExecutionOptions(InstalledAgentOptions): thinking: Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] = "high" tools: str = "read,bash,edit,write,update_plan" - subagents: Literal["disabled", "enabled"] = "disabled" + # The pinned CLI does not propagate prompt/context flags to child agents. + subagents: Literal["disabled"] = "disabled" project_resources: Literal["context-only"] = "context-only" background_process_lifetime: Literal["environment"] = "environment" # Arbitrary native config and prompt templates could defeat trial isolation diff --git a/evals/terminal-bench/src/diffex_terminal_bench/gate.py b/evals/terminal-bench/src/diffex_terminal_bench/gate.py index dc820e373..23390fcde 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/gate.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/gate.py @@ -246,8 +246,8 @@ def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = raise ValueError("Retained instruction changed") if sha256(run / "system-prompt.txt") != effective["system_prompt_sha256"]: raise ValueError("System prompt hash mismatch") - if effective["release_agents_sha256"] != identity["release_agents_sha256"]: - raise ValueError("Release harness identity mismatch") + if effective["system_prompt_sha256"] != identity["system_prompt_sha256"]: + raise ValueError("Eval system prompt identity mismatch") for name in ("stderr.log", "launcher.stdout.log", "launcher.stderr.log", "sessions/root.jsonl"): (run / name).read_bytes() validator = TrajectoryValidator() diff --git a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs index 30bd88114..e2ccb486e 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs @@ -17,8 +17,6 @@ export default function observe(api) { const prompt = typeof event.message.content === "string" ? event.message.content : event.message.content.filter((block) => block.type === "text").map((block) => block.text).join(""); const systemPrompt = ctx.getSystemPrompt(); - const releaseAgentsPath = process.env.DIFFEX_TB_RELEASE_AGENTS; - const releaseAgents = readFileSync(releaseAgentsPath, "utf8"); writeFileSync(join(runDir, "system-prompt.txt"), systemPrompt, { mode: 0o600 }); writeFileSync(join(runDir, "effective.json"), JSON.stringify({ provider: ctx.model?.provider, model: ctx.model?.id, @@ -32,9 +30,7 @@ export default function observe(api) { instruction_sha256: createHash("sha256").update(instruction).digest("hex"), prompt_sha256: createHash("sha256").update(prompt).digest("hex"), system_prompt_sha256: createHash("sha256").update(systemPrompt).digest("hex"), - release_agents_path: releaseAgentsPath, - release_agents_sha256: createHash("sha256").update(releaseAgents).digest("hex"), - release_agents_loaded: systemPrompt.includes(releaseAgents), + context_files_disabled: process.argv.includes("--no-context-files"), }, null, 2), { mode: 0o600 }); }); } diff --git a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py index 56f4da02b..fbf7a905a 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py @@ -7,7 +7,7 @@ from harbor.agents.installed.base import AgentAuthenticationError, NonZeroAgentExitCodeError from diffex_terminal_bench.lifecycle import read_cleanup -from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX +from diffex_terminal_bench.execution import SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX class DiffexExecutionError(NonZeroAgentExitCodeError): @@ -141,10 +141,11 @@ def parse_outcome(directory: Path, interruption: str | None = None, effective = json.loads((directory / "effective.json").read_text()) if not isinstance(requested, dict) or not isinstance(effective, dict): raise ValueError("invalid_configuration") - matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime", "user_message_suffix")) + matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime", "user_message_suffix", "system_prompt_sha256")) matches = matches and requested["user_message_suffix"] == USER_MESSAGE_SUFFIX + matches = matches and requested["system_prompt_sha256"] == SYSTEM_PROMPT_SHA256 matches = matches and requested.get("background_process_lifetime") == "environment" - matches = matches and effective.get("release_agents_loaded") is True and effective.get("instruction_delivered") is True + matches = matches and effective.get("context_files_disabled") is True and effective.get("instruction_delivered") is True matches = matches and sorted(effective["tools"]) == sorted(requested["tools"]) except (OSError, ValueError, KeyError, TypeError): matches = False diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs index 3494fff43..b3f750c66 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs @@ -1,6 +1,6 @@ // Launch the installed CLI with only the explicitly selected environment. const { spawn } = require("node:child_process"); -const { mkdirSync, mkdtempSync, readFileSync, writeFileSync, openSync, closeSync, existsSync, copyFileSync } = require("node:fs"); +const { mkdirSync, mkdtempSync, readFileSync, writeFileSync, openSync, closeSync, existsSync } = require("node:fs"); const { join } = require("node:path"); const { identity } = require("./cleanup.cjs"); @@ -17,14 +17,6 @@ writeFileSync(join(runDir, "state.json"), JSON.stringify({ state_dir: state, hom mkdirSync(join(state, "home"), { mode: 0o700 }); mkdirSync(join(runDir, "sessions"), { mode: 0o700 }); writeFileSync(join(state, "settings.json"), JSON.stringify({ evolution: { enabled: false } }), { mode: 0o600 }); -const bundledAgentsPath = "/opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist/AGENTS.md"; -let releaseAgentsPath = bundledAgentsPath; -if (!existsSync(bundledAgentsPath)) { - // 0.2.2 predates the packaged default context. Supply the exact release - // instructions in global scope; never shadow a newer packaged default. - releaseAgentsPath = join(state, "AGENTS.md"); - copyFileSync(join(runDir, "release-agents.md"), releaseAgentsPath); -} const env = { PATH: process.env.PATH, HOME: join(state, "home"), @@ -33,7 +25,6 @@ const env = { DIFFEX_SESSION_DIR: join(runDir, "sessions"), DIFFEX_OFFLINE: "1", DIFFEX_TB_RUN_DIR: runDir, - DIFFEX_TB_RELEASE_AGENTS: releaseAgentsPath, }; for (const name of config.credential_env_names) env[name] = process.env[name]; const args = [ @@ -44,6 +35,7 @@ const args = [ "--background-process-lifetime", config.background_process_lifetime, "--background-process-ownership-dir", join(runDir, "ownership"), "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", + "--no-context-files", "--system-prompt", join(runDir, "eval-system-prompt.txt"), "--extension", join(runDir, "observe.mjs"), "--print", `@${join(runDir, "instruction.txt")}`, config.user_message_suffix, ]; diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs index 05fbe8ba7..955002f5c 100644 --- a/evals/terminal-bench/tests/cleanup.test.cjs +++ b/evals/terminal-bench/tests/cleanup.test.cjs @@ -156,7 +156,6 @@ if (process.argv[2] === "service") { writeFileSync(join(f.runDir, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment", user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task." })); - writeFileSync(join(f.runDir, "release-agents.md"), "Fixture context"); const launcher = spawn(process.execPath, ["--require", join(__dirname, "fixtures/kill-before-cli-publication.cjs"), join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir], { env: { ...process.env, DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "off" }, diff --git a/evals/terminal-bench/tests/launcher.test.cjs b/evals/terminal-bench/tests/launcher.test.cjs index e4451df8a..90cb679c8 100644 --- a/evals/terminal-bench/tests/launcher.test.cjs +++ b/evals/terminal-bench/tests/launcher.test.cjs @@ -19,7 +19,6 @@ test("launcher preserves task packages outside exported logs and isolates each r credential_env_names: [], background_process_lifetime: "environment", user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task.", })); - writeFileSync(join(logs, "release-agents.md"), "fixture context"); const child = new EventEmitter(); child.pid = 12345; const gate = new EventEmitter(); @@ -33,6 +32,8 @@ test("launcher preserves task packages outside exported logs and isolates each r childEnv = options.env; assert.equal(command, "/bin/sh"); assert.ok(args.includes(join(logs, "ownership"))); + assert.ok(args.includes("--no-context-files")); + assert.equal(args[args.indexOf("--system-prompt") + 1], join(logs, "eval-system-prompt.txt")); assert.equal(args.at(-2), `@${join(logs, "instruction.txt")}`); assert.equal(args.at(-1), "\nDo not cheat by using online solutions or hints specific to this task."); return child; diff --git a/evals/terminal-bench/tests/test_execution.py b/evals/terminal-bench/tests/test_execution.py index 59b5f20ec..f07681665 100644 --- a/evals/terminal-bench/tests/test_execution.py +++ b/evals/terminal-bench/tests/test_execution.py @@ -18,7 +18,7 @@ from diffex_terminal_bench.benchmark import PACKAGE, run_identity, trial_config from diffex_terminal_bench.config import PINS from diffex_terminal_bench.dataset import load_config -from diffex_terminal_bench.execution import USER_MESSAGE_SUFFIX +from diffex_terminal_bench.execution import SYSTEM_PROMPT, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX class ExecutionTests(unittest.IsolatedAsyncioTestCase): @@ -55,13 +55,13 @@ def make_environment(self): environment.exec.return_value = ExecResult(return_code=0, stdout="", stderr="") return environment - def test_run_identity_uses_pinned_release_context(self): + def test_run_identity_uses_eval_system_prompt(self): config = load_config(PACKAGE / "benchmark.toml") with patch("diffex_terminal_bench.benchmark.subprocess.check_output", side_effect=["", "a" * 40]), \ patch("diffex_terminal_bench.benchmark.sha256", return_value="new-source-digest"): identity = run_identity(config, {}) - self.assertEqual(identity["release_agents_sha256"], PINS["release_agents_sha256"]) - self.assertEqual(identity["harness_files"]["release-agents.md"], "new-source-digest") + self.assertEqual(identity["system_prompt_sha256"], SYSTEM_PROMPT_SHA256) + self.assertEqual(identity["harness_files"]["eval-system-prompt.txt"], "new-source-digest") async def test_prompt_environment_and_fresh_trial_paths(self): environment = self.make_environment() @@ -82,6 +82,10 @@ async def upload(source, target): await agent.run(instruction, environment, AgentContext()) prompts = [value.decode() for key, value in uploads.items() if key.endswith("instruction.txt")] self.assertEqual(prompts, [instruction, instruction]) + system_prompts = [value.decode() for key, value in uploads.items() if key.endswith("eval-system-prompt.txt")] + self.assertEqual(system_prompts, [SYSTEM_PROMPT, SYSTEM_PROMPT]) + self.assertNotIn("", SYSTEM_PROMPT) + self.assertNotIn(" Date: Thu, 24 Sep 2026 10:37:56 -0400 Subject: [PATCH 3/3] fix(coding-agent): retry sandbox startup --- evals/terminal-bench/ARTIFACTS.md | 9 + .../diffex_terminal_bench/daytona_backend.py | 84 +++++++- .../tests/test_daytona_backend.py | 5 +- .../tests/test_startup_retries.py | 181 ++++++++++++++++++ 4 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 evals/terminal-bench/tests/test_startup_retries.py diff --git a/evals/terminal-bench/ARTIFACTS.md b/evals/terminal-bench/ARTIFACTS.md index cc75949f2..4007764cb 100644 --- a/evals/terminal-bench/ARTIFACTS.md +++ b/evals/terminal-bench/ARTIFACTS.md @@ -7,6 +7,15 @@ The backend waits up to 120 seconds for volume readiness and records its name, I The file retains a `mounts` list with session and environment names, so separate verifiers and restarted environments do not replace the agent's recovery location. Harbor's Docker-in-Docker backend binds the corresponding `/logs` subdirectories from the sandbox host into the task container. +Sandbox creation allows an initial attempt plus three startup-timeout retries, with deadlines of 60, 60, 60, and 300 seconds. +There is no retry backoff; each timed-out attempt first receives up to 20 seconds for sandbox lookup and deletion before the next create starts. +The adapter uses one unique sandbox name across these attempts to recover accepted creates after lost responses and prevent duplicate allocation from late requests. +`sandbox-startup.json` records attempt deadlines, outcomes, sandbox names, and cleanup results. +Failed cleanup stops retries and retains any known sandbox reference for Harbor's final cleanup. +Cancellation and non-timeout provider errors do not trigger creation retries. +The fourth creation timeout raises `EnvironmentStartTimeoutError`; Harbor's overall environment deadline still applies, including volume preparation and cleanup overhead. +These retries occur before agent execution and do not rerun evaluation attempts or change `--max-retries 0`. + Agent home, installed task packages, caches, and mutable harness state live in a private `/tmp/diffex-tb-state-*` directory on sandbox disk. They remain available to task services through verification and disappear with sandbox deletion. The exported `state.json` records these paths without linking the state tree into the logs. diff --git a/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py b/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py index 98b8a3186..216467a06 100644 --- a/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py +++ b/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py @@ -6,8 +6,9 @@ from uuid import uuid4 from daytona import AsyncDaytona, CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams, ListSandboxesQuery, VolumeMount -from daytona.common.errors import DaytonaConflictError, DaytonaNotFoundError +from daytona.common.errors import DaytonaConflictError, DaytonaNotFoundError, DaytonaTimeoutError from harbor.environments.daytona import DaytonaEnvironment +from harbor.trial.errors import EnvironmentStartTimeoutError from diffex_terminal_bench.gate import read_json, write_json from diffex_terminal_bench.transport import retry_transport @@ -56,7 +57,86 @@ async def get_volume(): "mount_path": mount.mount_path, "subpath": mount.subpath, }) write_json(path, record) - await super()._create_sandbox(params=params, daytona=daytona) + await self._create_with_retries(params, daytona) + + async def _create_with_retries( + self, params: CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams, + daytona: AsyncDaytona, + ) -> None: + # Use a stable, unique name to recover an accepted create whose SDK + # response timed out. Reusing it also prevents late requests from + # allocating multiple sandboxes for this creation sequence. + params = params.model_copy(deep=True) + params.name = params.name or f"diffex-start-{uuid4().hex}" + params.labels = {**(params.labels or {}), **self._sandbox_labels()} + path = self.trial_paths.trial_dir / "sandbox-startup.json" + record = read_json(path) if path.exists() else {"attempts": []} + for attempt, timeout in enumerate((60, 60, 60, 300), start=1): + entry = {"session_id": self.session_id, "name": params.name, + "attempt": attempt, "timeout_sec": timeout, "status": "starting"} + record["attempts"].append(entry) + write_json(path, record) + self.logger.info("Creating sandbox: attempt %s/4, timeout %ss", attempt, timeout) + try: + async with asyncio.timeout(timeout): + self._sandbox = await daytona.create(params=params, timeout=timeout) + except (TimeoutError, DaytonaTimeoutError) as exc: + entry["status"] = "timed_out" + await self._cleanup_start_attempt(daytona, params.name, entry, path, record) + if attempt == 4: + raise EnvironmentStartTimeoutError( + "Sandbox creation timed out after four attempts (60, 60, 60, 300 seconds)" + ) from exc + except BaseException: + # Cancellation and permanent provider errors must not start a + # replacement sandbox. Recover any accepted create first. + entry["status"] = "failed" + try: + await self._cleanup_start_attempt(daytona, params.name, entry, path, record) + except Exception: + self.logger.exception("Could not clean up failed sandbox creation %s", params.name) + raise + else: + entry.update(status="started", sandbox_id=self._sandbox.id) + write_json(path, record) + return + + async def _cleanup_start_attempt(self, daytona: AsyncDaytona, name: str, + entry: dict, path: Path, record: dict) -> None: + async def cleanup(): + async with asyncio.timeout(20): + try: + sandbox = await daytona.get(name, request_timeout=10) + except DaytonaNotFoundError: + entry["cleanup"] = "absent" + return + # Retain the reference if deletion fails so Harbor stop() can + # retry it. Never create a replacement after failed cleanup. + self._sandbox = sandbox + entry["sandbox_id"] = sandbox.id + try: + await daytona.delete(sandbox, wait=True, timeout=10) + except DaytonaNotFoundError: + pass + self._sandbox = None + entry["cleanup"] = "deleted" + + task = asyncio.create_task(cleanup()) + cancelled = False + try: + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + cancelled = True + task.result() + except Exception as exc: + entry.update(cleanup="failed", cleanup_error=type(exc).__name__) + raise + finally: + write_json(path, record) + if cancelled: + raise asyncio.CancelledError async def start(self, force_build: bool) -> None: try: diff --git a/evals/terminal-bench/tests/test_daytona_backend.py b/evals/terminal-bench/tests/test_daytona_backend.py index c1afd9d54..875e8b794 100644 --- a/evals/terminal-bench/tests/test_daytona_backend.py +++ b/evals/terminal-bench/tests/test_daytona_backend.py @@ -10,7 +10,6 @@ from daytona import CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams, Image from daytona.common.errors import DaytonaConflictError, DaytonaError -from harbor.environments.daytona import DaytonaEnvironment from harbor.models.job.config import JobConfig from harbor.models.task.config import EnvironmentConfig from harbor.models.trial.paths import TrialPaths @@ -31,7 +30,7 @@ def setUp(self): ) self.volume = SimpleNamespace(id="volume-id", state="ready") self.client = SimpleNamespace(volume=SimpleNamespace(get=AsyncMock(return_value=self.volume))) - parent = patch.object(DaytonaEnvironment, "_create_sandbox", new_callable=AsyncMock) + parent = patch.object(RecordedDaytonaEnvironment, "_create_with_retries", new_callable=AsyncMock) self.create = parent.start() self.addCleanup(parent.stop) @@ -97,7 +96,7 @@ async def test_dind_client_manager_and_custom_volume(self): params = CreateSandboxFromImageParams(image=Image.base("docker:28.3.3-dind")) await self.environment._create_sandbox(params) self.client.volume.get.assert_awaited_once_with("dedicated-project-logs", create=True) - self.create.assert_awaited_once_with(params=params, daytona=self.client) + self.create.assert_awaited_once_with(params, self.client) def test_native_full_job_selects_volume_backend(self): config = JobConfig.model_validate(yaml.safe_load((Path(__file__).resolve().parents[1] / "full.yaml").read_text())) diff --git a/evals/terminal-bench/tests/test_startup_retries.py b/evals/terminal-bench/tests/test_startup_retries.py new file mode 100644 index 000000000..79adcebd0 --- /dev/null +++ b/evals/terminal-bench/tests/test_startup_retries.py @@ -0,0 +1,181 @@ +"""Exercise startup retries through Harbor's actual direct startup path.""" + +import asyncio +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from daytona import CreateSandboxFromSnapshotParams +from daytona.common.errors import DaytonaAuthenticationError, DaytonaNotFoundError, DaytonaTimeoutError +from harbor.models.task.config import EnvironmentConfig +from harbor.models.trial.paths import TrialPaths +from harbor.trial.errors import EnvironmentStartTimeoutError + +from diffex_terminal_bench.daytona_backend import RecordedDaytonaEnvironment + + +class StartupRetryTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + (self.root / "Dockerfile").write_text("FROM ubuntu:24.04\n") + self.environment = RecordedDaytonaEnvironment( + environment_dir=self.root, environment_name="fixture", session_id="fixture", + trial_paths=TrialPaths(self.root / "trial"), task_env_config=EnvironmentConfig(), + ) + self.sandbox = SimpleNamespace(id="sandbox-id", snapshot="fixture", cpu=1, + memory=1, disk=10, labels={}) + self.client = SimpleNamespace( + create=AsyncMock(return_value=self.sandbox), + get=AsyncMock(side_effect=DaytonaNotFoundError("absent")), + delete=AsyncMock(), + volume=SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace(id="volume-id", state="ready"))), + ) + self.params = CreateSandboxFromSnapshotParams(snapshot="fixture") + manager = SimpleNamespace(get_client=AsyncMock(return_value=self.client)) + mock = patch("harbor.environments.daytona.environment.DaytonaClientManager.get_instance", + new_callable=AsyncMock, return_value=manager) + mock.start() + self.addCleanup(mock.stop) + for name, value in (("_configure_daytona_client", None), ("_resolve_start_sandbox_params", self.params), + ("ensure_dirs", None), ("_upload_environment_dir_after_start", None), + ("_write_sandbox_id_file", None)): + mock = patch.object(self.environment, name, new_callable=AsyncMock, return_value=value) + mock.start() + self.addCleanup(mock.stop) + + async def test_three_timeouts_then_fourth_attempt_succeeds(self): + self.client.create.side_effect = [DaytonaTimeoutError("startup timed out") for _ in range(3)] + [self.sandbox] + await self.environment.start(force_build=False) + self.assertEqual([call.kwargs["timeout"] for call in self.client.create.call_args_list], [60, 60, 60, 300]) + self.assertIs(self.environment._sandbox, self.sandbox) + record = json.loads((self.root / "trial/sandbox-created.json").read_text()) + self.assertEqual(record["sandboxes"][0]["id"], "sandbox-id") + + async def test_first_success_does_not_retry_or_mutate_caller_name(self): + await self.environment.start(force_build=False) + self.client.create.assert_awaited_once() + request = self.client.create.call_args.kwargs + self.assertEqual(request["timeout"], 60) + self.assertTrue(request["params"].name.startswith("diffex-start-")) + self.assertIsNone(self.params.name) + self.assertEqual(request["params"].snapshot, "fixture") + self.assertEqual(request["params"].labels["harbor.session_id"], "fixture") + self.client.get.assert_not_awaited() + + async def test_four_timeouts_raise_startup_timeout_and_keep_attempt_evidence(self): + self.client.create.side_effect = DaytonaTimeoutError("startup timed out") + with self.assertRaisesRegex(EnvironmentStartTimeoutError, "60, 60, 60, 300"): + await self.environment.start(force_build=False) + self.assertEqual(self.client.create.await_count, 4) + self.assertEqual(self.client.get.await_count, 4) + record = json.loads((self.root / "trial/sandbox-startup.json").read_text()) + self.assertEqual([row["timeout_sec"] for row in record["attempts"]], [60, 60, 60, 300]) + self.assertEqual({row["status"] for row in record["attempts"]}, {"timed_out"}) + self.assertEqual({row["cleanup"] for row in record["attempts"]}, {"absent"}) + self.assertEqual(len({row["name"] for row in record["attempts"]}), 1) + self.assertIsNone(self.environment._sandbox) + + async def test_actual_deadlines_cancel_stalled_create_and_allow_longer_last_attempt(self): + calls = [] + cancelled = [] + + async def create(*, params, timeout): + calls.append(timeout) + try: + # The real local deadline, rather than a synthetic SDK error, + # must stop each stalled attempt and grant the last more time. + await asyncio.sleep(0.15) + return self.sandbox + except asyncio.CancelledError: + cancelled.append(timeout) + raise + + self.client.create.side_effect = create + real_timeout = asyncio.timeout + with patch("diffex_terminal_bench.daytona_backend.asyncio.timeout", + side_effect=lambda seconds: real_timeout(seconds / 1000)): + await self.environment.start(force_build=False) + self.assertEqual(calls, [60, 60, 60, 300]) + self.assertEqual(cancelled, [60, 60, 60]) + + async def test_accepted_sandbox_is_deleted_before_replacement(self): + events = [] + + async def create(*, params, timeout): + events.append("create") + if events.count("create") == 1: + raise DaytonaTimeoutError("started remotely but response timed out") + return self.sandbox + + async def delete(sandbox, *, wait, timeout): + self.assertIs(sandbox, self.sandbox) + self.assertTrue(wait) + events.append("delete") + + self.client.create.side_effect = create + self.client.get.side_effect = None + self.client.get.return_value = self.sandbox + self.client.delete.side_effect = delete + await self.environment.start(force_build=False) + self.assertEqual(events, ["create", "delete", "create"]) + self.assertEqual(len({call.kwargs["params"].name for call in self.client.create.call_args_list}), 1) + + async def test_failed_cleanup_prevents_replacement_and_retains_sandbox(self): + self.client.create.side_effect = DaytonaTimeoutError("startup timed out") + self.client.get.side_effect = None + self.client.get.return_value = self.sandbox + self.client.delete.side_effect = RuntimeError("delete failed") + with self.assertRaisesRegex(RuntimeError, "delete failed"): + await self.environment.start(force_build=False) + self.assertEqual(self.client.create.await_count, 1) + self.assertIs(self.environment._sandbox, self.sandbox) + attempts = json.loads((self.root / "trial/sandbox-startup.json").read_text())["attempts"] + self.assertEqual(attempts[0]["cleanup"], "failed") + self.assertTrue((self.root / "trial/sandbox-created.json").is_file()) + + async def test_permanent_error_is_not_retried_or_replaced_by_cleanup_error(self): + self.client.create.side_effect = DaytonaAuthenticationError("unauthorized") + self.client.get.side_effect = DaytonaAuthenticationError("cleanup unauthorized") + with self.assertLogs(self.environment.logger, level="ERROR"): + with self.assertRaisesRegex(DaytonaAuthenticationError, "^unauthorized$"): + await self.environment.start(force_build=False) + self.assertEqual(self.client.create.await_count, 1) + + async def test_cancellation_cleans_up_without_starting_another_attempt(self): + started = asyncio.Event() + deleting = asyncio.Event() + release = asyncio.Event() + + async def create(**kwargs): + started.set() + await asyncio.Event().wait() + + async def delete(*args, **kwargs): + deleting.set() + await release.wait() + + self.client.create.side_effect = create + self.client.get.side_effect = None + self.client.get.return_value = self.sandbox + self.client.delete.side_effect = delete + task = asyncio.create_task(self.environment.start(force_build=False)) + await asyncio.wait_for(started.wait(), 1) + task.cancel() + await asyncio.wait_for(deleting.wait(), 1) + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, 1) + self.assertEqual(self.client.create.await_count, 1) + self.assertIsNone(self.environment._sandbox) + + +if __name__ == "__main__": + unittest.main()