From c1a9d6a9bf6135145e9a0b151b7a9362558dd8a7 Mon Sep 17 00:00:00 2001 From: kangzeyu Date: Thu, 5 Nov 2026 20:08:55 +0800 Subject: [PATCH 1/2] fix: remove local API key pooling and retry HTTP 429 Use configured API credentials directly and remove shared key state and selection options. Retry HTTP 429 with the existing bounded retry and Retry-After handling. Validation: ruff passed; pytest 621 passed, 1 skipped. A real rjob received HTTP 200 through the OSWorld2.1 entry and SlowModelClient; connectivity only, with the response truncated by the token limit. --- README.md | 30 +--- docs/CLUSTER_RUNTIME.md | 2 +- infra/config.example.json | 12 -- scripts/run_experiment.sh | 8 +- src/tandem/models/key_pool.py | 211 --------------------------- src/tandem/models/slow.py | 81 +++++----- src/tandem_eval/__main__.py | 10 -- src/tandem_eval/config_schema.py | 26 +--- src/tandem_eval/rjob_spec.py | 12 -- tests/eval/test_experiment_script.py | 2 - tests/harness/test_slow_transport.py | 112 ++++++++++---- 11 files changed, 136 insertions(+), 370 deletions(-) delete mode 100644 src/tandem/models/key_pool.py diff --git a/README.md b/README.md index abf90a9..3f30480 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ tandem-eval rjob run --run-id --benchmark osworld2 --tasks \ bash scripts/run_experiment.sh \ --experiment kimi12-retry-20261030 \ --tasks 024,029,037,045,057,059,069,082,088,093,005,031 \ - --concurrency 12 --slow-model kimi-k3 --slow-key-ids 1,2,3,4 \ + --concurrency 12 --slow-model kimi-k3 \ --max-steps 500 --max-wall 14400 ``` @@ -243,30 +243,16 @@ bash scripts/run_experiment.sh \ `drivers/.driver.log` 和 `-/`(worker、模型、轨迹与结果)。 worker 的 adpc 挂载只读时仍使用 evobox 暂存,任务结束后立即归档到该实验目录。 -集群内使用模板配置时,`api_key` 留空会使用默认号池,也可以手动填写自己的 `api_key`。 - -需要指定号池中的 key 时,在原启动命令后追加: - -```bash ---slow-key-ids 1,2,3,4 -``` - -也可以在所选慢模型条目的 `key_pool` 中配置,命令行参数优先: - -```json -{"key_ids": [1, 2, 3, 4]} -``` - -编号对应凭据列表中的第 1、2、3、4 把 key,不是密钥内容: - -- 不填编号或配置 `[]`:使用默认优先级。 -- 填一个编号:只使用这一把。 -- 填多个编号:这些 key 按同级轮询分配给 worker,正常请求固定使用分配到的 key;故障仅在指定集合内切换。 - -不同实验可指定不重叠的编号集合。这里均分的是 worker,不保证请求量严格相等;编号选择与手填完整 `api_key` 二选一。 +慢模型支持自定义 OpenAI 兼容 API。在 `harness.slow_model.apis` 的条目中填写 +服务地址 `base_url`、访问密钥 `api_key` 和模型名称 `model`。 +客户端以非流式方式向 `/chat/completions` 发送 POST 请求, +使用 `Authorization: Bearer ` 鉴权;`base_url` 可带尾部斜杠。 +真实密钥只填写在被 gitignore 的 `infra/config.json`;模板中的密钥留空。 慢模型的调用纪律(`models/slow.py`): +- HTTP 408、429、5xx 最多调用 3 次(首次 + 2 次重试),优先遵循 `Retry-After`, + 缺失时每次等待 30 秒;等待和请求均受任务剩余预算约束。其他 API 错误直接上报。 - 思考强度取所选 API 的配置;空正文(全留在思考通道)→ 下一次重试自动升档; - 叙述文字包裹 JSON → 自动剥围栏取首个平衡 `{…}` 抢救; - `finish_reason=length` → 明确报错,提示调大 `slow_max_tokens`。 diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 8c88099..2ce90ed 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -324,7 +324,7 @@ worker 侧各段日志: 每次实验必须指定唯一名称(小写字母、数字、连字符),所有结果归档到 `rjob_logs//`: -- `manifest.json`:tasks、并发、模型、key_ids、预算、watch timeout、git describe; +- `manifest.json`:tasks、并发、模型、预算、watch timeout、git describe; - `batch.log`:批次输出; - `drivers/.driver.log`:单个 rjob 生命周期日志; - `-/`:worker、模型调用、轨迹、截图和评分结果。 diff --git a/infra/config.example.json b/infra/config.example.json index d1248c1..87e2b88 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -7,12 +7,6 @@ "base_url": "https://api.minimax.cn/v1", "api_key": "", "model": "MiniMax-M3.1-Flash", - "key_pool": { - "provider": "minimax", - "key_ids": [], - "credentials_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/credentials.json", - "state_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/state.json" - }, "max_tokens": 16384, "timeout_seconds": 180, "reasoning_effort": "max", @@ -22,12 +16,6 @@ "base_url": "https://api.kimi.com/coding/v1", "api_key": "", "model": "k3.1-pro", - "key_pool": { - "provider": "kimi", - "key_ids": [], - "credentials_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/credentials.json", - "state_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/state.json" - }, "max_tokens": 16384, "timeout_seconds": 120, "reasoning_effort": null, diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh index 5bce418..54518dd 100755 --- a/scripts/run_experiment.sh +++ b/scripts/run_experiment.sh @@ -10,7 +10,6 @@ TASKS="" CONCURRENCY=4 SLOW_MODEL="kimi-k3" FAST_MODEL="" -SLOW_KEY_IDS="1,2,3,4" MAX_STEPS=500 MAX_WALL=14400 EXECUTION_MODE=stage @@ -22,7 +21,7 @@ usage() { Usage: bash scripts/run_experiment.sh \ --experiment NAME --tasks ID1,ID2,... [--concurrency N] \ - [--slow-model NAME] [--fast-model NAME] [--slow-key-ids 1,2,3,4] \ + [--slow-model NAME] [--fast-model NAME] \ [--execution-mode step|stage|goal] [--stage-max-actions N] \ [--max-steps N] [--max-wall SECONDS] [--dry-run] @@ -38,7 +37,6 @@ while [[ $# -gt 0 ]]; do --concurrency) CONCURRENCY="$2"; shift 2 ;; --slow-model) SLOW_MODEL="$2"; shift 2 ;; --fast-model) FAST_MODEL="$2"; shift 2 ;; - --slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;; --max-steps) MAX_STEPS="$2"; shift 2 ;; --max-wall) MAX_WALL="$2"; shift 2 ;; --execution-mode) EXECUTION_MODE="$2"; shift 2 ;; @@ -87,7 +85,7 @@ import sys print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"]) PY ) -export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL +export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL MAX_STEPS MAX_WALL export EXECUTION_MODE STAGE_MAX_ACTIONS FAST_MODEL python3 - <<'PY' import json @@ -103,7 +101,6 @@ args = { "concurrency": int(os.environ["CONCURRENCY"]), "slow_model": os.environ["SLOW_MODEL"], "fast_model": os.environ["FAST_MODEL"] or None, - "slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")], "max_steps": int(os.environ["MAX_STEPS"]), "max_wall_seconds": float(os.environ["MAX_WALL"]), "execution_mode": os.environ["EXECUTION_MODE"], @@ -149,7 +146,6 @@ run_task() { --stage-max-actions "$STAGE_MAX_ACTIONS" \ --slow-model "$SLOW_MODEL" \ "${fast_args[@]}" \ - --slow-key-ids "$SLOW_KEY_IDS" \ --max-steps "$MAX_STEPS" \ --max-wall "$MAX_WALL" \ --poll 60 \ diff --git a/src/tandem/models/key_pool.py b/src/tandem/models/key_pool.py deleted file mode 100644 index b8b1a2e..0000000 --- a/src/tandem/models/key_pool.py +++ /dev/null @@ -1,211 +0,0 @@ -"""慢模型 API key 的优先级选择与跨 worker 冷却;凭据不进入共享状态。""" - -from __future__ import annotations - -import fcntl -import json -import os -import tempfile -import time -from collections.abc import Iterator -from contextlib import contextmanager -from email.utils import parsedate_to_datetime -from pathlib import Path - -POOL_BASE_URLS = { - "minimax": "https://api.minimax.cn/v1", - "kimi": "https://api.kimi.com/coding/v1", -} - - -class PoolExhausted(RuntimeError): - """当前请求已没有可用的 key。""" - - -def validate_key_ids(key_ids) -> None: - if (not isinstance(key_ids, list) - or any(type(number) is not int or number < 1 for number in key_ids) - or len(set(key_ids)) != len(key_ids)): - raise ValueError("key_ids must be a list of distinct positive integers") - - -def classify_api_error(provider: str, status: int, code, message: str) -> tuple[str, float]: - """返回错误类别与 key 冷却秒数;零冷却的错误不切 key。""" - code = str(code) - if status == 401 or (provider == "minimax" and code in {"1004", "2049"}): - return "invalid_key", 3600 - if provider == "minimax": - if code in {"1008", "2053", "2056", "2067"}: - return "quota_exhausted", 300 - if code in {"1002", "1041", "1045", "2045", "2046", "2047", "2062"}: - return "rate_limited", 60 - if code in {"2022", "2057", "2061", "2068"}: - return "model_permission", 3600 - if code in {"1000", "1001", "1024", "1033", "2064", "2066"}: - return "transient", 0 - if provider == "kimi": - text = message.lower() - if status == 429 and any(part in text for part in ( - "usage limit", "credit balance is insufficient", "extra usage balance", - )): - return "quota_exhausted", 300 - if status == 429 and "engine is currently overloaded" in text: - return "transient", 0 - if status == 403 and any(part in text for part in ( - "subscription does not have access", "current plan supports only", - "unable to verify your membership benefits", - )): - return "model_permission", 3600 - if status == 429: - return "rate_limited", 60 - if status >= 500 or status == 408: - return "transient", 0 - return "fatal", 0 - - -def retry_after_seconds(value: str | None, default: float) -> float: - """Retry-After 支持秒数或 HTTP 日期;缺失时用本地冷却策略。""" - if value is None: - return default - try: - return max(0.0, float(value)) - except ValueError: - try: - return max(0.0, parsedate_to_datetime(value).timestamp() - time.time()) - except (TypeError, ValueError): - return default - - -class ApiKeyPool: - def __init__(self, credentials: dict): - base_url = credentials["base_url"].rstrip("/") - self.provider = next( - (name for name, url in POOL_BASE_URLS.items() if base_url == url), "custom" - ) - self.state_file: Path | None = None - self._local_state: dict = {} - self._preferred_id: str | None = None - self._selection_group = "" - options = credentials.get("key_pool") or {} - key_ids = options.get("key_ids", []) - validate_key_ids(key_ids) - if credentials.get("api_key") and key_ids: - raise ValueError("manual api_key cannot be combined with key_pool.key_ids") - # 手填 key 完全独立;无号池时也保留客户端原有的无鉴权 endpoint 用法。 - if credentials.get("api_key") or not options: - self.keys = [{"id": "manual", "priority": 1, "api_key": credentials.get("api_key", "")}] - self._secrets = [row["api_key"] for row in self.keys if row["api_key"]] - return - provider = options["provider"] - if provider not in POOL_BASE_URLS or POOL_BASE_URLS[provider] != base_url: - raise ValueError("key_pool provider does not match its official base_url") - self.keys = json.loads(Path(options["credentials_file"]).read_text(encoding="utf-8"))[provider] - if not isinstance(self.keys, list) or not self.keys: - raise ValueError("key_pool must contain a nonempty credential list") - for row in self.keys: - if not isinstance(row, dict) or not isinstance(row.get("id"), str) or not row["id"]: - raise ValueError("key_pool entries require a nonempty id") - if type(row.get("priority")) is not int or row["priority"] < 1: - raise ValueError("key_pool priorities must be positive integers") - key = row.get("api_key") - if not isinstance(key, str) or not key or key != key.strip(): - raise ValueError("key_pool contains an empty or whitespace-padded api_key") - if len({row["id"] for row in self.keys}) != len(self.keys): - raise ValueError("key_pool contains duplicate ids") - if len({row["api_key"] for row in self.keys}) != len(self.keys): - raise ValueError("key_pool contains duplicate keys") - self._secrets = [row["api_key"] for row in self.keys] - self.state_file = Path(options["state_file"]) - if not self.state_file.is_absolute() or not Path(options["credentials_file"]).is_absolute(): - raise ValueError("key_pool paths must be absolute") - if key_ids: - if max(key_ids) > len(self.keys): - raise ValueError("key_pool.key_ids contains an out-of-range credential number") - self.keys = [self.keys[number - 1] for number in sorted(key_ids)] - self._selection_group = ",".join(str(number) for number in sorted(key_ids)) - with self._state() as state: - cursors = state.setdefault("worker_cursors", {}) - available = self._available(state) or self.keys - cursor = cursors.get(self._selection_group, 0) - self._preferred_id = available[cursor % len(available)]["id"] - cursors[self._selection_group] = (cursor + 1) % len(available) - - @contextmanager - def _state(self) -> Iterator[dict]: - if self.state_file is None: - yield self._local_state - return - parent = self.state_file.parent - parent.mkdir(mode=0o700, parents=True, exist_ok=True) - owner = parent.stat() - descriptor = os.open(str(self.state_file) + ".lock", os.O_CREAT | os.O_RDWR, 0o600) - with os.fdopen(descriptor, "r+") as lock: - # rjob worker 以 root 运行时,私有状态仍归目录所有者管理。 - if os.geteuid() == 0: - os.fchown(lock.fileno(), owner.st_uid, owner.st_gid) - fcntl.flock(lock, fcntl.LOCK_EX) - state = json.loads(self.state_file.read_text()) if self.state_file.exists() else {} - yield state.setdefault(self.provider, {}) - descriptor, temporary = tempfile.mkstemp(prefix=self.state_file.name + ".", dir=parent) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - if os.geteuid() == 0: - os.fchown(stream.fileno(), owner.st_uid, owner.st_gid) - json.dump(state, stream, ensure_ascii=False) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, self.state_file) - finally: - Path(temporary).unlink(missing_ok=True) - - def _available(self, state: dict, excluded: set[str] | None = None) -> list[dict]: - excluded = excluded or set() - now = time.time() - cooldowns = state.get("cooldowns", {}) - return [row for row in self.keys if row["id"] not in excluded - and cooldowns.get(row["id"], {}).get("until", 0) <= now] - - def select(self, excluded: set[str] | None = None) -> dict: - with self._state() as state: - cooldowns = state.get("cooldowns", {}) - available = self._available(state, excluded) - if not available: - next_retry = min( - (cooldowns.get(row["id"], {}).get("until", 0) for row in self.keys), default=0 - ) - raise PoolExhausted( - f"pool_exhausted: provider={self.provider}, next_retry_at={next_retry}" - ) - if self._preferred_id is not None: - preferred = next((row for row in available if row["id"] == self._preferred_id), None) - if preferred is not None: - return preferred - tier = self.keys - cursor_name = "selected:" + self._selection_group - else: - priority = min(row["priority"] for row in available) - tier = [row for row in self.keys if row["priority"] == priority] - cursor_name = str(priority) - cursors = state.setdefault("cursors", {}) - cursor = cursors.get(cursor_name, 0) - indices = ((cursor + offset) % len(tier) for offset in range(len(tier))) - index = next(index for index in indices if tier[index] in available) - cursors[cursor_name] = (index + 1) % len(tier) - return tier[index] - - def cooldown(self, key_id: str, seconds: float, reason: str) -> None: - with self._state() as state: - cooldowns = state.setdefault("cooldowns", {}) - previous = cooldowns.get(key_id, {}).get("until", 0) - cooldowns[key_id] = {"until": max(previous, time.time() + seconds), "reason": reason} - - def redact(self, value): - if isinstance(value, str): - for secret in self._secrets: - value = value.replace(secret, "[REDACTED]") - return value - if isinstance(value, dict): - return {key: self.redact(item) for key, item in value.items()} - if isinstance(value, list): - return [self.redact(item) for item in value] - return value diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py index 0d569a3..bd0a9b4 100644 --- a/src/tandem/models/slow.py +++ b/src/tandem/models/slow.py @@ -12,12 +12,11 @@ import time import urllib.error import urllib.request +from email.utils import parsedate_to_datetime from pathlib import Path from PIL import Image -from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds - # 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败 _REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"} @@ -131,13 +130,25 @@ def _api_failure(status: int, raw: dict): return base.get("status_code"), str(error or base.get("status_msg") or json.dumps(raw)) +def _retry_after_seconds(value: str | None, default: float) -> float: + """Honor the endpoint's Retry-After without maintaining account state.""" + if value is None: + return default + try: + return max(0.0, float(value)) + except ValueError: + try: + return max(0.0, parsedate_to_datetime(value).timestamp() - time.time()) + except (TypeError, ValueError): + return default + + class SlowModelClient: def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): if isinstance(credential_file, dict): self.credentials = dict(credential_file) else: self.credentials = json.loads(Path(credential_file).read_text(encoding="utf-8")) - self.key_pool = ApiKeyPool(self.credentials) self._call_id = 0 self.config = config self.audit_path = audit_path @@ -146,6 +157,16 @@ def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): self.llm_sink = llm_sink self.deadline = None + def _redact(self, value): + if isinstance(value, str): + secret = self.credentials.get("api_key") + return value.replace(secret, "[REDACTED]") if secret else value + if isinstance(value, dict): + return {key: self._redact(item) for key, item in value.items()} + if isinstance(value, list): + return [self._redact(item) for item in value] + return value + def set_deadline(self, deadline): """Absolute monotonic task deadline, shared by requirements, planning and verification.""" self.deadline = deadline @@ -200,8 +221,8 @@ def build_request(): data=data, headers={ **( - {"Authorization": "Bearer " + key["api_key"]} - if key["api_key"] + {"Authorization": "Bearer " + credentials.get("api_key", "")} + if credentials.get("api_key", "") else {} ), "Content-Type": "application/json", @@ -218,11 +239,11 @@ def audit(event, started, attempt, **details): record = { "event": event, "provider": "slow_model", - "api_provider": self.key_pool.provider, + "api_provider": "custom", "role": role, "call_id": call_id, - "key_id": key["id"], - "key_priority": key["priority"], + "key_id": "manual", + "key_priority": 1, "attempt": attempt, "time": time.time(), "elapsed_seconds": round(time.perf_counter() - started, 3), @@ -230,7 +251,7 @@ def audit(event, started, attempt, **details): **image_stats, **details, } - text = json.dumps(self.key_pool.redact(record), ensure_ascii=False) + text = json.dumps(self._redact(record), ensure_ascii=False) print(text, flush=True) with self.audit_path.open("a", encoding="utf-8") as log: log.write(text + "\n") @@ -250,19 +271,6 @@ def public_headers(headers): } return {name.lower(): value for name, value in headers.items() if name.lower() in allowed} - excluded: set[str] = set() - - def alternate_key(category, seconds, started, attempt): - if len(self.key_pool.keys) == 1: - return key # A manually configured endpoint keeps its bounded retry path. - self.key_pool.cooldown(key["id"], seconds, category) - excluded.add(key["id"]) - audit("key_cooldown", started, attempt, category=category, cooldown_seconds=seconds) - replacement = self.key_pool.select(excluded) - audit("key_changed", started, attempt, category=category, next_key_id=replacement["id"]) - return replacement - - key = self.key_pool.select() attempt = 0 failures = 0 while True: @@ -280,7 +288,7 @@ def alternate_key(category, seconds, started, attempt): except urllib.error.HTTPError as error: status = error.code headers = public_headers(error.headers) - body = self.key_pool.redact(error.read().decode("utf-8", errors="replace")) + body = self._redact(error.read().decode("utf-8", errors="replace")) try: raw = json.loads(body) except ValueError: @@ -290,40 +298,28 @@ def alternate_key(category, seconds, started, attempt): if self.llm_sink is not None: self.llm_sink( role=role, - request=self.key_pool.redact({"payload": payload, "messages": messages}), - response=self.key_pool.redact(raw), + request=self._redact({"payload": payload, "messages": messages}), + response=self._redact(raw), attempt=attempt, call_id=call_id, - key_id=key["id"], key_priority=key["priority"], + key_id="manual", key_priority=1, http_status=status, elapsed_seconds=round(time.perf_counter() - started, 3), ) failure = _api_failure(status, raw) if failure is not None: code, message = failure - category, cooldown = classify_api_error(self.key_pool.provider, status, code, message) + category = "transient" if status >= 500 or status in {408, 429} else "fatal" audit("api_error", started, attempt, http_status=status, headers=headers, error_code=code, category=category, error=message[:2000], usage=raw.get("usage")) - error = RuntimeError(self.key_pool.redact( + error = RuntimeError(self._redact( f"slow model API error: status={status}, code={code}, message={message[:2000]}" )) - if cooldown: - cooldown = retry_after_seconds(headers.get("retry-after"), cooldown) - self.key_pool.cooldown(key["id"], cooldown, category) - excluded.add(key["id"]) - audit("key_cooldown", started, attempt, category=category, - cooldown_seconds=cooldown) - try: - key = self.key_pool.select(excluded) - except PoolExhausted as exhausted: - raise exhausted from error - continue if category != "transient": raise error failures += 1 if failures >= 3: raise error - delay = retry_after_seconds(headers.get("retry-after"), 30) - key = alternate_key("transient", max(60, delay), started, attempt) + delay = _retry_after_seconds(headers.get("retry-after"), 30) audit("retry_wait", started, attempt, retry_wait_seconds=delay) self._retry_wait(delay) continue @@ -362,7 +358,7 @@ def alternate_key(category, seconds, started, attempt): started, attempt, error_type=type(error).__name__, - error=self.key_pool.redact(str(error))[:1000], + error=self._redact(str(error))[:1000], ) failures += 1 if failures >= 3: @@ -373,7 +369,6 @@ def alternate_key(category, seconds, started, attempt): http.client.IncompleteRead)): if isinstance(error, TaskBudgetExceeded): raise - key = alternate_key("transport_error", 60, started, attempt) elif (self.config.execution_mode == "goal" and role == "plan_stage" and "finish_reason=length" in str(error)): payload["reasoning_effort"] = "low" diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index 8dad50e..0947fc6 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -41,8 +41,6 @@ def main(argv: list[str] | None = None) -> int: help="阶段内动作数上限,达到后请求剩余工作的重规划") run_cmd.add_argument("--slow-model", default=None, metavar="NAME", help="慢模型(大脑)API 名:config 的 harness.slow_model.apis 里选;默认取 default") - run_cmd.add_argument("--slow-key-ids", default=None, metavar="1,2,3", - help="号池 key 编号;多个编号按 worker 均分,未指定则取配置") run_cmd.add_argument("--fast-model", default=None, metavar="NAME", help="快模型(小脑)API 名:config 的 harness.fast_model.apis 里选;默认取 default") run_cmd.add_argument("--slow-max-tokens", type=int, default=None, @@ -78,14 +76,6 @@ def main(argv: list[str] | None = None) -> int: config = load_config(args.config) if args.rjob_command == "run": overrides: dict = {"benchmark": args.benchmark} - if args.slow_key_ids is not None: - try: - overrides["slow_key_ids"] = ( - [int(item.strip()) for item in args.slow_key_ids.split(",")] - if args.slow_key_ids.strip() else [] - ) - except ValueError: - parser.error("--slow-key-ids 需填写正整数编号,多个编号用逗号分隔") manifest: dict = {} if args.tasks: manifest["tasks"] = [item.strip() for item in args.tasks.split(",") if item.strip()] diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index 6d696c1..cef65a3 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -20,8 +20,6 @@ from pathlib import Path from typing import Any -from tandem.models.key_pool import POOL_BASE_URLS, validate_key_ids - #: 必填路径(点号路径,逐段下钻;None / 空串 / 空列表视为缺失) REQUIRED_PATHS = ( "harness.slow_model.default", @@ -123,28 +121,8 @@ def _validate_slow_models(config: dict) -> list[str]: base_url = api.get("base_url") if isinstance(base_url, str) and base_url.strip() and not base_url.startswith("http"): problems.append(f"harness.slow_model.apis.{name}.base_url(应为 http(s) 地址)") - pool = api.get("key_pool") - if isinstance(pool, dict): - try: - validate_key_ids(pool.get("key_ids", [])) - except ValueError: - problems.append(f"harness.slow_model.apis.{name}.key_pool.key_ids(应为不重复的正整数列表)") - if api.get("api_key") and pool.get("key_ids"): - problems.append(f"harness.slow_model.apis.{name}(api_key 与 key_ids 不能同时指定)") - if _is_missing(api.get("api_key")): - prefix = f"harness.slow_model.apis.{name}.key_pool" - if not isinstance(pool, dict): - problems.append(f"harness.slow_model.apis.{name}.api_key / .key_pool(至少给一个)") - else: - provider = pool.get("provider") - if provider not in POOL_BASE_URLS: - problems.append(f"{prefix}.provider(应为 minimax / kimi)") - elif not isinstance(base_url, str) or base_url.rstrip("/") != POOL_BASE_URLS[provider]: - problems.append(f"{prefix}.provider 与官方 base_url 不匹配") - for key in ("credentials_file", "state_file"): - path = pool.get(key) - if not isinstance(path, str) or not path.startswith("/"): - problems.append(f"{prefix}.{key}(必须是绝对路径)") + if not isinstance(api.get("api_key", ""), str): + problems.append(f"harness.slow_model.apis.{name}.api_key(应为字符串)") # 思考强度必须落在 harness 认识的档位里,否则 SessionConfig 会在运行期才报错 effort = api.get("reasoning_effort") if effort is not None and effort not in REASONING_EFFORTS: diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 0e8a1fc..c9fff1d 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -20,8 +20,6 @@ import subprocess from pathlib import Path -from tandem.models.key_pool import validate_key_ids - from .config_schema import select_api @@ -79,8 +77,6 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic }, "registry": config["eval"].get("registry"), } - if not slow_api.get("api_key") and slow_api.get("key_pool"): - secrets["slow_model"]["key_pool"] = dict(slow_api["key_pool"]) if benchmark == "osworld21": section = config["eval"]["benchmarks"][benchmark] environment = {} @@ -167,14 +163,6 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, requested = dict(overrides.pop("models", {}) or {}) slow_name, slow_api = select_api(config, "slow_model", requested.get("slow")) fast_name, fast_api = select_api(config, "fast_model", requested.get("fast")) - key_ids = overrides.pop("slow_key_ids", None) - if key_ids is not None: - validate_key_ids(key_ids) - if key_ids and (slow_api.get("api_key") or not slow_api.get("key_pool")): - raise ValueError("slow_key_ids requires key_pool without a manual api_key") - if slow_api.get("key_pool"): - slow_api["key_pool"] = {**slow_api["key_pool"], "key_ids": key_ids} - inner_image = config["eval"]["inner_image"] if benchmark == "osworld21": # OSW2.1 用发布清单固定的运行镜像(digest),不覆盖 latest,也不复用 OSW2 的镜像 diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py index 9773b32..a7cdd9c 100644 --- a/tests/eval/test_experiment_script.py +++ b/tests/eval/test_experiment_script.py @@ -17,7 +17,6 @@ def test_dry_run_creates_manifest_and_batch_log(tmp_path): "--tasks", "024,029", "--concurrency", "12", "--slow-model", "kimi-k3", - "--slow-key-ids", "1,2,3,4", "--max-steps", "500", "--max-wall", "14400", "--dry-run", @@ -33,7 +32,6 @@ def test_dry_run_creates_manifest_and_batch_log(tmp_path): assert manifest["tasks"] == ["024", "029"] assert manifest["concurrency"] == 12 assert manifest["slow_model"] == "kimi-k3" - assert manifest["slow_key_ids"] == [1, 2, 3, 4] assert manifest["max_steps"] == 500 assert manifest["max_wall_seconds"] == 14400.0 assert manifest["watch_timeout_seconds"] == 15000.0 diff --git a/tests/harness/test_slow_transport.py b/tests/harness/test_slow_transport.py index 753dc26..957ddd6 100644 --- a/tests/harness/test_slow_transport.py +++ b/tests/harness/test_slow_transport.py @@ -200,25 +200,19 @@ def opener(request, **kwargs): @pytest.fixture -def pooled_client(tmp_path, monkeypatch): - credentials = tmp_path / "synthetic-credentials.json" - credentials.write_text(json.dumps({"kimi": [ - {"id": f"synthetic-{i}", "priority": 1, "api_key": f"test-only-{i}"} - for i in range(3) - ]})) +def endpoint_client(tmp_path, monkeypatch): monkeypatch.setattr("tandem.models.slow.time.sleep", lambda seconds: None) return SlowModelClient({ - "base_url": "https://api.kimi.com/coding/v1", "model": "synthetic", - "key_pool": {"provider": "kimi", "key_ids": [1, 2, 3], - "credentials_file": str(credentials), "state_file": str(tmp_path / "state.json")}, + "base_url": "http://gateway/v1/", "model": "MiniMax-M3.1-Flash", + "api_key": "test-only-gateway-key", }, SessionConfig(slow_reasoning_effort="low"), tmp_path / "audit.jsonl") @pytest.mark.parametrize("role", ["plan_stage", "verify_stage", "ground_visual"]) @pytest.mark.parametrize("failure", [TimeoutError, http.client.RemoteDisconnected, urllib.error.URLError, http.client.IncompleteRead]) -def test_transport_failure_switches_key_without_changing_reasoning_or_replaying_actions( - pooled_client, monkeypatch, role, failure): +def test_transport_failure_retries_endpoint_without_changing_reasoning_or_replaying_actions( + endpoint_client, monkeypatch, role, failure): requests = [] def opener(request, **kwargs): @@ -230,18 +224,21 @@ def opener(request, **kwargs): return _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role=role) == {"done": True} - assert [r.get_header("Authorization") for r in requests] == ["Bearer test-only-0", "Bearer test-only-1"] + assert len(requests) == 2 + assert all(r.full_url == "http://gateway/v1/chat/completions" for r in requests) + assert all(r.get_header("Authorization") == "Bearer test-only-gateway-key" for r in requests) assert requests[0].data == requests[1].data - events = [json.loads(line) for line in pooled_client.audit_path.read_text().splitlines()] - changed = next(e for e in events if e["event"] == "key_changed") - assert (changed["key_id"], changed["next_key_id"]) == ("synthetic-0", "synthetic-1") - assert "test-only-" not in pooled_client.audit_path.read_text() - assert pooled_client.key_pool.select()["id"] != "synthetic-0" + payload = json.loads(requests[0].data) + assert payload["model"] == "MiniMax-M3.1-Flash" and payload["stream"] is False + events = [json.loads(line) for line in endpoint_client.audit_path.read_text().splitlines()] + assert not any(e["event"] in {"key_cooldown", "key_changed"} for e in events) + assert "test-only-gateway-key" not in endpoint_client.audit_path.read_text() + assert sorted(p.name for p in endpoint_client.audit_path.parent.iterdir()) == ["audit.jsonl"] -def test_temporary_server_error_switches_key(pooled_client, monkeypatch): +def test_temporary_server_error_retries_same_endpoint(endpoint_client, monkeypatch): requests = [] class Unavailable(_Response): @@ -255,13 +252,14 @@ def opener(request, **kwargs): return Unavailable() if len(requests) == 1 else _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") == {"done": True} - assert requests[0].get_header("Authorization") != requests[1].get_header("Authorization") + assert len(requests) == 2 + assert requests[0].get_header("Authorization") == requests[1].get_header("Authorization") @pytest.mark.parametrize("failure", [TimeoutError, http.client.IncompleteRead]) -def test_transport_key_rotation_is_bounded(pooled_client, monkeypatch, failure): +def test_endpoint_transport_retries_are_bounded(endpoint_client, monkeypatch, failure): requests = [] def opener(request, **kwargs): @@ -272,12 +270,13 @@ def opener(request, **kwargs): monkeypatch.setattr(urllib.request, "urlopen", opener) with pytest.raises(failure): - pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") - assert len(requests) == len({r.get_header("Authorization") for r in requests}) == 3 + assert len(requests) == 3 + assert len({r.get_header("Authorization") for r in requests}) == 1 -def test_invalid_json_does_not_cool_down_or_switch_key(pooled_client, monkeypatch): +def test_invalid_json_repair_uses_same_endpoint(endpoint_client, monkeypatch): requests = [] class Invalid(_Response): @@ -289,7 +288,66 @@ def opener(request, **kwargs): return Invalid() if len(requests) == 1 else _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") == {"done": True} assert requests[0].get_header("Authorization") == requests[1].get_header("Authorization") - assert "key_cooldown" not in pooled_client.audit_path.read_text() + assert "key_cooldown" not in endpoint_client.audit_path.read_text() + + +@pytest.mark.parametrize("status", [400, 401, 402, 403]) +def test_gateway_errors_are_redacted_without_local_account_state(endpoint_client, monkeypatch, status): + calls, captured = [], [] + secret = endpoint_client.credentials["api_key"] + endpoint_client.llm_sink = lambda **record: captured.append(record) + + def opener(request, **kwargs): + calls.append(request) + body = json.dumps({"error": {"message": "gateway rejected " + secret}}).encode() + raise urllib.error.HTTPError(request.full_url, status, "gateway error", {}, io.BytesIO(body)) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + with pytest.raises(RuntimeError, match=f"status={status}") as raised: + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") + assert len(calls) == 1 and captured[0]["http_status"] == status + assert secret not in str(raised.value) + endpoint_client.audit_path.read_text() + json.dumps(captured) + assert "[REDACTED]" in json.dumps(captured) + assert sorted(p.name for p in endpoint_client.audit_path.parent.iterdir()) == ["audit.jsonl"] + + +@pytest.mark.parametrize("status", [408, 429, 503]) +@pytest.mark.parametrize("headers,delay", [({}, 30), ({"Retry-After": "2"}, 2)]) +def test_gateway_retries_honor_retry_after_and_remain_bounded( + endpoint_client, monkeypatch, status, headers, delay): + calls, waits = [], [] + + def opener(request, **kwargs): + calls.append(request) + raise urllib.error.HTTPError(request.full_url, status, "busy", headers, + io.BytesIO(b'{"error":{"message":"busy"}}')) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError, match=f"status={status}"): + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") + assert len(calls) == 3 and waits == [delay, delay] + assert len({r.get_header("Authorization") for r in calls}) == 1 + + +def test_gateway_429_can_recover_on_last_attempt(endpoint_client, monkeypatch): + calls, waits = [], [] + + def opener(request, **kwargs): + calls.append(request) + if len(calls) < 3: + raise urllib.error.HTTPError(request.full_url, 429, "busy", {}, + io.BytesIO(b'{"error":{"message":"rate limited"}}')) + return _Response() + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") == {"done": True} + assert len(calls) == 3 and waits == [30, 30] + assert len({r.data for r in calls}) == 1 From 1ee2ae0aeaeed8aebdd48c6dd6c3fc3580ad7f90 Mon Sep 17 00:00:00 2001 From: kangzeyu Date: Thu, 5 Nov 2026 21:35:17 +0800 Subject: [PATCH 2/2] fix: preserve MiniMax HTTP 200 transient error retries Classify MiniMax base_resp transient error codes independently of credential management, preserving bounded retries for direct and proxied responses. Keep unrelated provider errors and non-retryable HTTP statuses fatal. Validation: ruff passed; pytest 657 passed, 1 skipped. A local HTTP fixture returned a MiniMax temporary error followed by a valid response, confirming retry recovery with the same request and credentials. --- README.md | 3 +- src/tandem/models/api_errors.py | 14 +++++ src/tandem/models/slow.py | 4 +- tests/harness/test_slow_api_errors.py | 88 +++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 src/tandem/models/api_errors.py create mode 100644 tests/harness/test_slow_api_errors.py diff --git a/README.md b/README.md index 3f30480..112ed19 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,8 @@ worker 的 adpc 挂载只读时仍使用 evobox 暂存,任务结束后立即 慢模型的调用纪律(`models/slow.py`): -- HTTP 408、429、5xx 最多调用 3 次(首次 + 2 次重试),优先遵循 `Retry-After`, +- HTTP 408、429、5xx,以及 MiniMax 通过 HTTP 200 返回的临时故障, + 最多调用 3 次(首次 + 2 次重试),优先遵循 `Retry-After`, 缺失时每次等待 30 秒;等待和请求均受任务剩余预算约束。其他 API 错误直接上报。 - 思考强度取所选 API 的配置;空正文(全留在思考通道)→ 下一次重试自动升档; - 叙述文字包裹 JSON → 自动剥围栏取首个平衡 `{…}` 抢救; diff --git a/src/tandem/models/api_errors.py b/src/tandem/models/api_errors.py new file mode 100644 index 0000000..029c234 --- /dev/null +++ b/src/tandem/models/api_errors.py @@ -0,0 +1,14 @@ +"""API error classification, independent of credentials and request scheduling.""" + +_MINIMAX_TRANSIENT_CODES = {"1000", "1001", "1024", "1033", "2064", "2066"} + + +def classify_api_error(status: int, raw: dict) -> str: + """Recognize transient HTTP errors and MiniMax's HTTP 200 error envelope.""" + if status >= 500 or status in {408, 429}: + return "transient" + base = raw.get("base_resp") + if (status == 200 and isinstance(base, dict) + and str(base.get("status_code")) in _MINIMAX_TRANSIENT_CODES): + return "transient" + return "fatal" diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py index bd0a9b4..159f60f 100644 --- a/src/tandem/models/slow.py +++ b/src/tandem/models/slow.py @@ -17,6 +17,8 @@ from PIL import Image +from .api_errors import classify_api_error + # 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败 _REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"} @@ -308,7 +310,7 @@ def public_headers(headers): failure = _api_failure(status, raw) if failure is not None: code, message = failure - category = "transient" if status >= 500 or status in {408, 429} else "fatal" + category = classify_api_error(status, raw) audit("api_error", started, attempt, http_status=status, headers=headers, error_code=code, category=category, error=message[:2000], usage=raw.get("usage")) error = RuntimeError(self._redact( diff --git a/tests/harness/test_slow_api_errors.py b/tests/harness/test_slow_api_errors.py new file mode 100644 index 0000000..56bcbba --- /dev/null +++ b/tests/harness/test_slow_api_errors.py @@ -0,0 +1,88 @@ +"""Provider response errors retain retry semantics without account state.""" + +import io +import json +import urllib.request + +import pytest + +from tandem.config import SessionConfig +from tandem.models.slow import SlowModelClient + + +class _Response(io.BytesIO): + status = 200 + headers = {} + + +@pytest.fixture +def client(tmp_path): + return SlowModelClient({ + "base_url": "https://api.minimax.cn/v1/", "api_key": "test-only-key", + "model": "MiniMax-M3.1-Flash", + }, SessionConfig(), tmp_path / "audit.jsonl") + + +@pytest.mark.parametrize("base_url", ["https://api.minimax.cn/v1/", "http://gateway/v1/"]) +@pytest.mark.parametrize("code_type", [int, str]) +@pytest.mark.parametrize("code", [1000, 1001, 1024, 1033, 2064, 2066]) +def test_minimax_http_200_temporary_error_recovers_with_same_request( + client, monkeypatch, base_url, code_type, code): + client.credentials["base_url"] = base_url + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + raw = ({"base_resp": {"status_code": code_type(code), "status_msg": "temporary failure"}} + if len(requests) == 1 else + {"choices": [{"message": {"content": '{"ok":true}'}, "finish_reason": "stop"}]}) + return _Response(json.dumps(raw).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + assert client.json_request([{"role": "user", "content": "probe"}], lambda v: v == {"ok": True}, + role="plan_stage") == {"ok": True} + assert len(requests) == 2 and waits == [30] + assert requests[0].data == requests[1].data + assert all(r.full_url == base_url.rstrip("/") + "/chat/completions" for r in requests) + assert all(r.get_header("Authorization") == "Bearer test-only-key" for r in requests) + assert sorted(p.name for p in client.audit_path.parent.iterdir()) == ["audit.jsonl"] + + +@pytest.mark.parametrize("code", [1000, 1001, 1024, 1033, 2064, 2066]) +def test_minimax_temporary_error_retries_remain_bounded(client, monkeypatch, code): + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + return _Response(json.dumps({"base_resp": {"status_code": code, "status_msg": "busy"}}).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError, match=f"code={code}"): + client.json_request([{"role": "user", "content": "probe"}], lambda v: True, role="plan_stage") + assert len(requests) == 3 and waits == [30, 30] + + +@pytest.mark.parametrize("status,raw", [ + (200, {"base_resp": {"status_code": 1004, "status_msg": "invalid key"}}), + (200, {"base_resp": {"status_code": 1008, "status_msg": "quota exhausted"}}), + (200, {"base_resp": {"status_code": 9999, "status_msg": "unknown failure"}}), + (200, {"error": {"code": 1000, "message": "unrelated provider error"}}), + (400, {"base_resp": {"status_code": 1000, "status_msg": "bad request"}}), + (401, {"base_resp": {"status_code": 1000, "status_msg": "unauthorized"}}), +]) +def test_other_api_errors_do_not_gain_retries(client, monkeypatch, status, raw): + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + response = _Response(json.dumps(raw).encode()) + response.status = status + return response + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError): + client.json_request([{"role": "user", "content": "probe"}], lambda v: True, role="plan_stage") + assert len(requests) == 1 and waits == []