From 52a2c16bde6dd967d6f430de9280353c97aed979 Mon Sep 17 00:00:00 2001 From: rwkeyes <86691029+rwkeyes@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:31:03 -0500 Subject: [PATCH] serve: honour conv_route_batch_after from the config, not only the flag The launcher writes "conv_route_batch_after" into the run config and starts the server without --conv-route-batch-after, while the server read only the flag (argparse default 4). Every deployment therefore ran 4 whatever the config said. Measured on 2026-11-14: two mixed-traffic windows whose configs differed only in that key (4 against 1) produced identical results on all eleven steps - a conversation's second turn 0.587 s in one arm and 0.555 s in the other, the single-shot steps matching to 0.02 s. Passing the flag by hand reproduced the 2026-11-09 finding at once (C2 2.465 s, a re-prefill). resolve_batch_after() in serve/conv_router.py resolves the value: explicit flag, then the config key, then 4; malformed values fall back rather than raise. The flag's argparse default becomes None so an absent flag is distinguishable from an explicit 4. A test pins all four cases. Not for dirigo/0.2.12-verified: that branch keeps serve/ byte-identical to the deployed snapshot by design, and the deployment is unaffected today (its config says 4). Land this with the next tree bump. --- serve/conv_router.py | 21 +++++++++++++++++++++ serve/server.py | 8 +++++--- serve/test_conv_router.py | 13 ++++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/serve/conv_router.py b/serve/conv_router.py index ac8f428607a..a1a0866825c 100644 --- a/serve/conv_router.py +++ b/serve/conv_router.py @@ -24,6 +24,27 @@ CHAT = "chat" # cache on BATCH = "batch" # cache off +DEFAULT_BATCH_AFTER = 4 + + +def resolve_batch_after(cli, cfg): + """The run length the router uses: the CLI flag wins, then the config's key, then 4. + + The launcher writes `conv_route_batch_after` into the run config and starts the server without the flag + (`strata-b70-serve.sh`), so a server that only read the flag ran 4 for every deployment - measured 2026-11-14: + a config of 1 and the default produced byte-identical mixed-traffic results until the flag was passed by hand. + """ + for v in (cli, (cfg or {}).get("conv_route_batch_after") if isinstance(cfg, dict) else None): + if v is None or isinstance(v, bool): + continue + try: + n = int(v) + except (TypeError, ValueError): + continue + if n >= 1: + return n + return DEFAULT_BATCH_AFTER + @dataclass class Decision: diff --git a/serve/server.py b/serve/server.py index 13d0e4907eb..7c7104d6f39 100644 --- a/serve/server.py +++ b/serve/server.py @@ -65,6 +65,7 @@ tool_choice_of, unmark_think_literals) from serve.mcp import McpCancelled, hub_from_config # noqa: E402 from serve import conv_router as conv_route_mod # noqa: E402 +from serve.conv_router import resolve_batch_after # noqa: E402 from serve import runconfig # noqa: E402 from serve.webfiles import subtree_file as web_subtree_file # noqa: E402 from strata_control import engine_component # noqa: E402 @@ -10386,8 +10387,9 @@ def notify_lifecycle(kind: str, payload: dict) -> None: help="switch the engine's prompt/conversation cache per traffic shape: single-shot batches off " "(prefill +33 %%), chat/agent on (a repeated context reuses, 5.3 s -> 0.3 s). 'shadow' only " "logs what it would do (serve/conv_router.py)") - ap.add_argument("--conv-route-batch-after", type=int, default=4, - help="consecutive single-shot requests before the cache is switched off (default 4)") + ap.add_argument("--conv-route-batch-after", type=int, default=None, + help="consecutive single-shot requests before the cache is switched off (default: the config's " + "conv_route_batch_after, else 4)") ap.add_argument("--thinking-policy", choices=["off", "shadow", "on"], default=None, help="do not pay for a thinking block on a request that asks for a short direct answer " "(extraction, formatting, translation, a one-word reply; serve/prompt_kind.py " @@ -10731,7 +10733,7 @@ def request_stop(reason: str): _pc = 6 _cr = cfg.get("conv_route") svc.set_conv_route(a.conv_route or (_cr if isinstance(_cr, str) else "") or "off", - cache_on=_pc, batch_after=a.conv_route_batch_after) + cache_on=_pc, batch_after=resolve_batch_after(a.conv_route_batch_after, cfg)) if (cfg.get("arch") or "") == "deepseek41" and vision is None and a.engine == "strata": # DeepSeek-V4.1 reads pictures with the engine's own tower (--vision on, the default): no separate reader process. # `--vision off` in the config's args, or a pack without a tower, leaves svc.vision None: pictures are refused with the 400 as before. diff --git a/serve/test_conv_router.py b/serve/test_conv_router.py index 41e68746627..1e10eaff861 100644 --- a/serve/test_conv_router.py +++ b/serve/test_conv_router.py @@ -8,7 +8,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from serve.conv_router import BATCH, CHAT, Router # noqa: E402 +from serve.conv_router import BATCH, CHAT, Router, resolve_batch_after # noqa: E402 def one(text="Summarise this document."): @@ -106,6 +106,17 @@ def test_report_counts_every_request(): assert rep["chat"] + rep["batch"] == 3 +def test_the_config_key_is_honoured_when_the_flag_is_absent(): + """The launcher writes conv_route_batch_after into the config and passes no flag, so the config must win over 4.""" + assert resolve_batch_after(None, {"conv_route_batch_after": 2}) == 2 + assert resolve_batch_after(None, {"conv_route_batch_after": "3"}) == 3 + assert resolve_batch_after(1, {"conv_route_batch_after": 2}) == 1, "an explicit flag wins over the config" + assert resolve_batch_after(None, {}) == 4, "no flag and no key keeps the old default" + assert resolve_batch_after(None, None) == 4 + for bad in (0, -1, True, "off", [1], {}): + assert resolve_batch_after(None, {"conv_route_batch_after": bad}) == 4, f"malformed {bad!r} must not crash" + + def test_the_prefix_history_is_bounded(): r = Router(repeat_window=8) for i in range(50):