From c64b8f3410c603a444de3cba98b89002f2e84867 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 30 Sep 2026 15:23:19 +0530 Subject: [PATCH 1/4] fix(harness): require everything=true before dropping every scenario --- src/fi/alk/harness/scenario_tools.py | 10 ++++++-- .../harness/skills/write-scenarios/SKILL.md | 2 +- tests/test_harness.py | 25 +++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index aea6b66eb..de13ce60b 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1952,12 +1952,18 @@ async def suite_progress(args: dict[str, Any]) -> dict[str, Any]: @tool( "drop_scenario", - "Remove a scenario by name, or all of them with name '*'.", - schema({"name": str}, ["name"]), + "Remove a scenario by name. Removing every scenario takes name '*' with " + "everything=true, and only when the person asked to start the suite over.", + schema({"name": str, "everything": bool}, ["name"]), ) async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: name = str(args.get("name") or "") if name == "*": + if args.get("everything") is not True: + return _err( + "Dropping every scenario needs everything=true. Do that only when the " + "person asked to start the suite over; otherwise drop scenarios by name." + ) kept.clear() write_scenarios(kept, destination, catalogue) return _ok("all scenarios dropped") diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 5a8bf336b..4c900a0ea 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -373,7 +373,7 @@ called. Assert on what the world looks like afterwards and on what the caller wa | `try_calls` | Runs calls against a **throwaway copy** of the world and shows the state they leave. This is how you work out a solution and what its checks should assert. Nothing you do here is visible to anybody else. | | `add_sub_goal` | Adds a named thing this agent can be checked on, with its check in code. | | `submit_scenario` | Keeps one scenario, after validation and the three gates. | -| `drop_scenario` | Removes one by name, or all of them with `*`. | +| `drop_scenario` | Removes one by name; all of them only with `*` and `everything=true`. | | `aim_for` | Sets how many scenarios are wanted. Needed when reopening an existing suite to add more, because the target starts at what is already there. Not for saving a suite nobody asked for. | | `save_scenarios` | Finishes the suite. Reports what is off about it as a whole. | | `amend_contract`, `add_rule`, `drop_rule`, `fix_tool` | Correct the contract when it is wrong. See the last section. | diff --git a/tests/test_harness.py b/tests/test_harness.py index ccfd2d4aa..5025c9923 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -7995,3 +7995,28 @@ def test_a_live_target_is_not_asked_to_seed_what_the_caller_is_told(): assert not any("to say back" in problem for problem in live) assert any("to say back" in problem for problem in owned) + + +def test_dropping_every_scenario_needs_an_explicit_everything(tmp_path): + import asyncio + + from mcp.types import CallToolRequestParams + + from fi.alk.harness import scenario_tools as module + + root, contract, _catalogue = _built_environment(tmp_path) + server, _kept = module.scenario_tools(contract, root, root, wanted=1) + instance = _instance(server) + + async def drop(arguments): + handler = _request_handler(instance, "tools/call") + answer = await handler.handler( + None, CallToolRequestParams(name="drop_scenario", arguments=arguments) + ) + return answer.isError, answer.content[0].text + + refused = asyncio.run(drop({"name": "*"})) + accepted = asyncio.run(drop({"name": "*", "everything": True})) + + assert refused[0] and "everything=true" in refused[1] + assert not accepted[0] and "all scenarios dropped" in accepted[1] From 7f0a604e7303f1baf45bca6f197cbb256be5707a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 30 Sep 2026 17:41:05 +0530 Subject: [PATCH 2/4] fix(harness): say an emptied suite is not saved until a scenario is submitted --- src/fi/alk/harness/scenario_tools.py | 13 +++++++++++-- src/fi/alk/harness/skills/write-scenarios/SKILL.md | 2 +- tests/test_harness.py | 14 +++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index a30c48534..eddf20e7a 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -251,6 +251,14 @@ def persona_vocabulary_note() -> str: ) +# A suite is never saved empty, so an emptied list changes nothing on disk until one is submitted. +_EMPTIED = ( + "Every scenario is dropped from this session's list, but nothing is saved yet: a suite is never " + "saved empty, so the saved scenarios stay until you submit at least one new scenario, which " + "then replaces them. Do not tell the person the suite is empty or saved." +) + + def write_scenarios( scenarios: list[Scenario], destination: Path, catalogue: Catalogue | None = None ) -> Path: @@ -1973,12 +1981,13 @@ async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: "person asked to start the suite over; otherwise drop scenarios by name." ) kept.clear() - write_scenarios(kept, destination, catalogue) - return _ok("all scenarios dropped") + return _ok(_EMPTIED) before = len(kept) kept[:] = [one for one in kept if one.name != name] if len(kept) == before: return _err(f"no scenario called {name!r}") + if not kept: + return _ok(_EMPTIED) write_scenarios(kept, destination, catalogue) return _ok(f"{name} dropped. {len(kept)} left") diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 4c900a0ea..2c48eb5a9 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -373,7 +373,7 @@ called. Assert on what the world looks like afterwards and on what the caller wa | `try_calls` | Runs calls against a **throwaway copy** of the world and shows the state they leave. This is how you work out a solution and what its checks should assert. Nothing you do here is visible to anybody else. | | `add_sub_goal` | Adds a named thing this agent can be checked on, with its check in code. | | `submit_scenario` | Keeps one scenario, after validation and the three gates. | -| `drop_scenario` | Removes one by name; all of them only with `*` and `everything=true`. | +| `drop_scenario` | Removes one by name; all of them only with `*` and `everything=true`. A suite is never saved empty: the saved scenarios stay until you submit a new one, which replaces them. | | `aim_for` | Sets how many scenarios are wanted. Needed when reopening an existing suite to add more, because the target starts at what is already there. Not for saving a suite nobody asked for. | | `save_scenarios` | Finishes the suite. Reports what is off about it as a whole. | | `amend_contract`, `add_rule`, `drop_rule`, `fix_tool` | Correct the contract when it is wrong. See the last section. | diff --git a/tests/test_harness.py b/tests/test_harness.py index 5025c9923..2a4682da4 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -8004,7 +8004,12 @@ def test_dropping_every_scenario_needs_an_explicit_everything(tmp_path): from fi.alk.harness import scenario_tools as module - root, contract, _catalogue = _built_environment(tmp_path) + from fi.alk.harness.scenario_tools import accept_scenario + + root, contract, catalogue = _built_environment(tmp_path) + saved = [] + accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=saved) + module.write_scenarios(saved, root, catalogue) server, _kept = module.scenario_tools(contract, root, root, wanted=1) instance = _instance(server) @@ -8016,7 +8021,10 @@ async def drop(arguments): return answer.isError, answer.content[0].text refused = asyncio.run(drop({"name": "*"})) - accepted = asyncio.run(drop({"name": "*", "everything": True})) + by_name = asyncio.run(drop({"name": saved[0].name})) + everything = asyncio.run(drop({"name": "*", "everything": True})) assert refused[0] and "everything=true" in refused[1] - assert not accepted[0] and "all scenarios dropped" in accepted[1] + assert not by_name[0] and "nothing is saved yet" in by_name[1] + assert not everything[0] and "nothing is saved yet" in everything[1] + assert [one.name for one in module.load_scenarios(root)] == [saved[0].name] From 45046e864f6d3050a557ce2d7430c1e6d4e73fa2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 30 Sep 2026 14:57:57 +0530 Subject: [PATCH 3/4] fix(harness): apply an edited max_turns to the scenario a run plays --- src/fi/alk/harness/call_runner.py | 2 +- tests/harness/test_scenario_edits.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 653772103..26f8fb250 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -447,7 +447,7 @@ def _read_scenario_document(bundle_dir: Path, scenario_key: str) -> dict[str, An # Edits a call actually plays; everything else is as sealed. -_EDITABLE_FIELDS = ("background_noise",) +_EDITABLE_FIELDS = ("background_noise", "max_turns") _EDITABLE_PERSONA_FIELDS = ( "personality", "communication_style", diff --git a/tests/harness/test_scenario_edits.py b/tests/harness/test_scenario_edits.py index 21423d461..d1300b6c7 100644 --- a/tests/harness/test_scenario_edits.py +++ b/tests/harness/test_scenario_edits.py @@ -18,9 +18,14 @@ "scenario_edits": { "book-ride": { "background_noise": "street", + "max_turns": 4, "tests": "unused by a call", "instruction": "ignored", - "persona": {"personality": "impatient", "name": "ignored"}, + "persona": { + "personality": "impatient", + "languages": ["English", "Hindi"], + "name": "ignored", + }, } } } @@ -30,7 +35,13 @@ def test_edits_replace_only_what_a_call_plays(): merged = _with_scenario_edits(SEALED, EDITS, "book-ride") assert merged["background_noise"] == "street" - assert merged["persona"] == {"name": "Asha", "accent": "Indian", "personality": "impatient"} + assert merged["persona"] == { + "name": "Asha", + "accent": "Indian", + "personality": "impatient", + "languages": ["English", "Hindi"], + } + assert merged["max_turns"] == 4 assert merged["tests"] == "books the ride" assert merged["instruction"] == "book a ride" assert SEALED["background_noise"] == "office" @@ -47,7 +58,10 @@ def test_a_voice_call_plays_the_edited_place(): def test_a_chat_caller_is_the_edited_persona(): merged = _with_scenario_edits(SEALED, EDITS, "book-ride") - assert _conversation_scenario(merged).persona.personality == "impatient" + scenario = _conversation_scenario(merged) + + assert scenario.persona.personality == "impatient" + assert scenario.max_turns == 4 def test_no_edits_leave_the_sealed_document(): From 63a1e59fa715b2e0be1b7c1c1351ce485a51e885 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 9 Oct 2026 09:57:33 +0530 Subject: [PATCH 4/4] fix(harness): import fi.simulate on the callback adapter's main thread --- src/fi/alk/harness/callback_http_adapter.py | 5 +- tests/harness/test_callback_http_adapter.py | 73 +++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 tests/harness/test_callback_http_adapter.py diff --git a/src/fi/alk/harness/callback_http_adapter.py b/src/fi/alk/harness/callback_http_adapter.py index 39c4fa820..e78fa969e 100644 --- a/src/fi/alk/harness/callback_http_adapter.py +++ b/src/fi/alk/harness/callback_http_adapter.py @@ -17,6 +17,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any +# Main thread only: with livekit installed this import registers plugins. +from fi.simulate import AgentInput + def _load_callback() -> Any: target = os.environ["ALK_CALLBACK_ENTRYPOINT"] @@ -46,8 +49,6 @@ def _json_body(value: Any) -> dict[str, Any]: def _invoke(payload: dict[str, Any]) -> dict[str, Any]: - from fi.simulate import AgentInput - result = CALLBACK(AgentInput.model_validate(payload)) if inspect.isawaitable(result): result = asyncio.run(result) diff --git a/tests/harness/test_callback_http_adapter.py b/tests/harness/test_callback_http_adapter.py new file mode 100644 index 000000000..cb21eb3ae --- /dev/null +++ b/tests/harness/test_callback_http_adapter.py @@ -0,0 +1,73 @@ +import json +import os +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +import pytest + +ADAPTER = ( + Path(__file__).resolve().parents[2] / "src/fi/alk/harness/callback_http_adapter.py" +) + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +@pytest.fixture +def adapter(tmp_path): + (tmp_path / "chat_agent.py").write_text( + "def reply(agent_input):\n" + " return {'content': 'thread ' + agent_input.thread_id}\n", + encoding="utf-8", + ) + # Bundles embed the adapter's source and run it from the bundle, not from the package. + script = tmp_path / "adapter.py" + script.write_text(ADAPTER.read_text(encoding="utf-8"), encoding="utf-8") + port = _free_port() + process = subprocess.Popen( + [sys.executable, str(script)], + env={ + **os.environ, + "ALK_CALLBACK_ENTRYPOINT": "chat_agent:reply", + "PORT": str(port), + "PYTHONPATH": str(tmp_path), + }, + stderr=subprocess.PIPE, + ) + base = f"http://127.0.0.1:{port}" + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + urllib.request.urlopen(f"{base}/health", timeout=1) + break + except OSError: + if process.poll() is not None: + pytest.fail(process.stderr.read().decode()) + time.sleep(0.2) + yield base + process.terminate() + process.wait(timeout=10) + + +def test_a_callback_answers_from_the_request_thread(adapter): + """The request thread is not the main thread; nothing imported there may require it.""" + body = json.dumps( + { + "thread_id": "t1", + "messages": [{"role": "user", "content": "hello"}], + "new_message": {"role": "user", "content": "hello"}, + } + ).encode() + request = urllib.request.Request( + f"{adapter}/invoke", data=body, headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(request, timeout=30) as response: + assert response.status == 200 + assert json.loads(response.read())["content"] == "thread t1"