diff --git a/README.md b/README.md index abf90a9..112ed19 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ tandem-eval rjob run --run-id --benchmark osworld2 --tasks \ bash scripts/run_experiment.sh \ --experiment kimi12-retry-20261030 \ --tasks 024,029,037,045,057,059,069,082,088,093,005,031 \ - --concurrency 12 --slow-model kimi-k3 --slow-key-ids 1,2,3,4 \ + --concurrency 12 --slow-model kimi-k3 \ --max-steps 500 --max-wall 14400 ``` @@ -243,30 +243,17 @@ bash scripts/run_experiment.sh \ `drivers/.driver.log` 和 `-/`(worker、模型、轨迹与结果)。 worker 的 adpc 挂载只读时仍使用 evobox 暂存,任务结束后立即归档到该实验目录。 -集群内使用模板配置时,`api_key` 留空会使用默认号池,也可以手动填写自己的 `api_key`。 - -需要指定号池中的 key 时,在原启动命令后追加: - -```bash ---slow-key-ids 1,2,3,4 -``` - -也可以在所选慢模型条目的 `key_pool` 中配置,命令行参数优先: - -```json -{"key_ids": [1, 2, 3, 4]} -``` - -编号对应凭据列表中的第 1、2、3、4 把 key,不是密钥内容: - -- 不填编号或配置 `[]`:使用默认优先级。 -- 填一个编号:只使用这一把。 -- 填多个编号:这些 key 按同级轮询分配给 worker,正常请求固定使用分配到的 key;故障仅在指定集合内切换。 - -不同实验可指定不重叠的编号集合。这里均分的是 worker,不保证请求量严格相等;编号选择与手填完整 `api_key` 二选一。 +慢模型支持自定义 OpenAI 兼容 API。在 `harness.slow_model.apis` 的条目中填写 +服务地址 `base_url`、访问密钥 `api_key` 和模型名称 `model`。 +客户端以非流式方式向 `/chat/completions` 发送 POST 请求, +使用 `Authorization: Bearer ` 鉴权;`base_url` 可带尾部斜杠。 +真实密钥只填写在被 gitignore 的 `infra/config.json`;模板中的密钥留空。 慢模型的调用纪律(`models/slow.py`): +- HTTP 408、429、5xx,以及 MiniMax 通过 HTTP 200 返回的临时故障, + 最多调用 3 次(首次 + 2 次重试),优先遵循 `Retry-After`, + 缺失时每次等待 30 秒;等待和请求均受任务剩余预算约束。其他 API 错误直接上报。 - 思考强度取所选 API 的配置;空正文(全留在思考通道)→ 下一次重试自动升档; - 叙述文字包裹 JSON → 自动剥围栏取首个平衡 `{…}` 抢救; - `finish_reason=length` → 明确报错,提示调大 `slow_max_tokens`。 diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 8c88099..2ce90ed 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -324,7 +324,7 @@ worker 侧各段日志: 每次实验必须指定唯一名称(小写字母、数字、连字符),所有结果归档到 `rjob_logs//`: -- `manifest.json`:tasks、并发、模型、key_ids、预算、watch timeout、git describe; +- `manifest.json`:tasks、并发、模型、预算、watch timeout、git describe; - `batch.log`:批次输出; - `drivers/.driver.log`:单个 rjob 生命周期日志; - `-/`:worker、模型调用、轨迹、截图和评分结果。 diff --git a/infra/config.example.json b/infra/config.example.json index d1248c1..87e2b88 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -7,12 +7,6 @@ "base_url": "https://api.minimax.cn/v1", "api_key": "", "model": "MiniMax-M3.1-Flash", - "key_pool": { - "provider": "minimax", - "key_ids": [], - "credentials_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/credentials.json", - "state_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/state.json" - }, "max_tokens": 16384, "timeout_seconds": 180, "reasoning_effort": "max", @@ -22,12 +16,6 @@ "base_url": "https://api.kimi.com/coding/v1", "api_key": "", "model": "k3.1-pro", - "key_pool": { - "provider": "kimi", - "key_ids": [], - "credentials_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/credentials.json", - "state_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/secrets/tandem_api_pool/state.json" - }, "max_tokens": 16384, "timeout_seconds": 120, "reasoning_effort": null, diff --git a/scripts/run_experiment.sh b/scripts/run_experiment.sh index 5bce418..54518dd 100755 --- a/scripts/run_experiment.sh +++ b/scripts/run_experiment.sh @@ -10,7 +10,6 @@ TASKS="" CONCURRENCY=4 SLOW_MODEL="kimi-k3" FAST_MODEL="" -SLOW_KEY_IDS="1,2,3,4" MAX_STEPS=500 MAX_WALL=14400 EXECUTION_MODE=stage @@ -22,7 +21,7 @@ usage() { Usage: bash scripts/run_experiment.sh \ --experiment NAME --tasks ID1,ID2,... [--concurrency N] \ - [--slow-model NAME] [--fast-model NAME] [--slow-key-ids 1,2,3,4] \ + [--slow-model NAME] [--fast-model NAME] \ [--execution-mode step|stage|goal] [--stage-max-actions N] \ [--max-steps N] [--max-wall SECONDS] [--dry-run] @@ -38,7 +37,6 @@ while [[ $# -gt 0 ]]; do --concurrency) CONCURRENCY="$2"; shift 2 ;; --slow-model) SLOW_MODEL="$2"; shift 2 ;; --fast-model) FAST_MODEL="$2"; shift 2 ;; - --slow-key-ids) SLOW_KEY_IDS="$2"; shift 2 ;; --max-steps) MAX_STEPS="$2"; shift 2 ;; --max-wall) MAX_WALL="$2"; shift 2 ;; --execution-mode) EXECUTION_MODE="$2"; shift 2 ;; @@ -87,7 +85,7 @@ import sys print(json.loads(Path(sys.argv[1]).read_text())["log"]["results_root"]) PY ) -export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL SLOW_KEY_IDS MAX_STEPS MAX_WALL +export STAGING EXPERIMENT DRIVER_DIR TASKS CONCURRENCY SLOW_MODEL MAX_STEPS MAX_WALL export EXECUTION_MODE STAGE_MAX_ACTIONS FAST_MODEL python3 - <<'PY' import json @@ -103,7 +101,6 @@ args = { "concurrency": int(os.environ["CONCURRENCY"]), "slow_model": os.environ["SLOW_MODEL"], "fast_model": os.environ["FAST_MODEL"] or None, - "slow_key_ids": [int(item) for item in os.environ["SLOW_KEY_IDS"].split(",")], "max_steps": int(os.environ["MAX_STEPS"]), "max_wall_seconds": float(os.environ["MAX_WALL"]), "execution_mode": os.environ["EXECUTION_MODE"], @@ -149,7 +146,6 @@ run_task() { --stage-max-actions "$STAGE_MAX_ACTIONS" \ --slow-model "$SLOW_MODEL" \ "${fast_args[@]}" \ - --slow-key-ids "$SLOW_KEY_IDS" \ --max-steps "$MAX_STEPS" \ --max-wall "$MAX_WALL" \ --poll 60 \ diff --git a/src/tandem/models/api_errors.py b/src/tandem/models/api_errors.py new file mode 100644 index 0000000..029c234 --- /dev/null +++ b/src/tandem/models/api_errors.py @@ -0,0 +1,14 @@ +"""API error classification, independent of credentials and request scheduling.""" + +_MINIMAX_TRANSIENT_CODES = {"1000", "1001", "1024", "1033", "2064", "2066"} + + +def classify_api_error(status: int, raw: dict) -> str: + """Recognize transient HTTP errors and MiniMax's HTTP 200 error envelope.""" + if status >= 500 or status in {408, 429}: + return "transient" + base = raw.get("base_resp") + if (status == 200 and isinstance(base, dict) + and str(base.get("status_code")) in _MINIMAX_TRANSIENT_CODES): + return "transient" + return "fatal" diff --git a/src/tandem/models/key_pool.py b/src/tandem/models/key_pool.py deleted file mode 100644 index b8b1a2e..0000000 --- a/src/tandem/models/key_pool.py +++ /dev/null @@ -1,211 +0,0 @@ -"""慢模型 API key 的优先级选择与跨 worker 冷却;凭据不进入共享状态。""" - -from __future__ import annotations - -import fcntl -import json -import os -import tempfile -import time -from collections.abc import Iterator -from contextlib import contextmanager -from email.utils import parsedate_to_datetime -from pathlib import Path - -POOL_BASE_URLS = { - "minimax": "https://api.minimax.cn/v1", - "kimi": "https://api.kimi.com/coding/v1", -} - - -class PoolExhausted(RuntimeError): - """当前请求已没有可用的 key。""" - - -def validate_key_ids(key_ids) -> None: - if (not isinstance(key_ids, list) - or any(type(number) is not int or number < 1 for number in key_ids) - or len(set(key_ids)) != len(key_ids)): - raise ValueError("key_ids must be a list of distinct positive integers") - - -def classify_api_error(provider: str, status: int, code, message: str) -> tuple[str, float]: - """返回错误类别与 key 冷却秒数;零冷却的错误不切 key。""" - code = str(code) - if status == 401 or (provider == "minimax" and code in {"1004", "2049"}): - return "invalid_key", 3600 - if provider == "minimax": - if code in {"1008", "2053", "2056", "2067"}: - return "quota_exhausted", 300 - if code in {"1002", "1041", "1045", "2045", "2046", "2047", "2062"}: - return "rate_limited", 60 - if code in {"2022", "2057", "2061", "2068"}: - return "model_permission", 3600 - if code in {"1000", "1001", "1024", "1033", "2064", "2066"}: - return "transient", 0 - if provider == "kimi": - text = message.lower() - if status == 429 and any(part in text for part in ( - "usage limit", "credit balance is insufficient", "extra usage balance", - )): - return "quota_exhausted", 300 - if status == 429 and "engine is currently overloaded" in text: - return "transient", 0 - if status == 403 and any(part in text for part in ( - "subscription does not have access", "current plan supports only", - "unable to verify your membership benefits", - )): - return "model_permission", 3600 - if status == 429: - return "rate_limited", 60 - if status >= 500 or status == 408: - return "transient", 0 - return "fatal", 0 - - -def retry_after_seconds(value: str | None, default: float) -> float: - """Retry-After 支持秒数或 HTTP 日期;缺失时用本地冷却策略。""" - if value is None: - return default - try: - return max(0.0, float(value)) - except ValueError: - try: - return max(0.0, parsedate_to_datetime(value).timestamp() - time.time()) - except (TypeError, ValueError): - return default - - -class ApiKeyPool: - def __init__(self, credentials: dict): - base_url = credentials["base_url"].rstrip("/") - self.provider = next( - (name for name, url in POOL_BASE_URLS.items() if base_url == url), "custom" - ) - self.state_file: Path | None = None - self._local_state: dict = {} - self._preferred_id: str | None = None - self._selection_group = "" - options = credentials.get("key_pool") or {} - key_ids = options.get("key_ids", []) - validate_key_ids(key_ids) - if credentials.get("api_key") and key_ids: - raise ValueError("manual api_key cannot be combined with key_pool.key_ids") - # 手填 key 完全独立;无号池时也保留客户端原有的无鉴权 endpoint 用法。 - if credentials.get("api_key") or not options: - self.keys = [{"id": "manual", "priority": 1, "api_key": credentials.get("api_key", "")}] - self._secrets = [row["api_key"] for row in self.keys if row["api_key"]] - return - provider = options["provider"] - if provider not in POOL_BASE_URLS or POOL_BASE_URLS[provider] != base_url: - raise ValueError("key_pool provider does not match its official base_url") - self.keys = json.loads(Path(options["credentials_file"]).read_text(encoding="utf-8"))[provider] - if not isinstance(self.keys, list) or not self.keys: - raise ValueError("key_pool must contain a nonempty credential list") - for row in self.keys: - if not isinstance(row, dict) or not isinstance(row.get("id"), str) or not row["id"]: - raise ValueError("key_pool entries require a nonempty id") - if type(row.get("priority")) is not int or row["priority"] < 1: - raise ValueError("key_pool priorities must be positive integers") - key = row.get("api_key") - if not isinstance(key, str) or not key or key != key.strip(): - raise ValueError("key_pool contains an empty or whitespace-padded api_key") - if len({row["id"] for row in self.keys}) != len(self.keys): - raise ValueError("key_pool contains duplicate ids") - if len({row["api_key"] for row in self.keys}) != len(self.keys): - raise ValueError("key_pool contains duplicate keys") - self._secrets = [row["api_key"] for row in self.keys] - self.state_file = Path(options["state_file"]) - if not self.state_file.is_absolute() or not Path(options["credentials_file"]).is_absolute(): - raise ValueError("key_pool paths must be absolute") - if key_ids: - if max(key_ids) > len(self.keys): - raise ValueError("key_pool.key_ids contains an out-of-range credential number") - self.keys = [self.keys[number - 1] for number in sorted(key_ids)] - self._selection_group = ",".join(str(number) for number in sorted(key_ids)) - with self._state() as state: - cursors = state.setdefault("worker_cursors", {}) - available = self._available(state) or self.keys - cursor = cursors.get(self._selection_group, 0) - self._preferred_id = available[cursor % len(available)]["id"] - cursors[self._selection_group] = (cursor + 1) % len(available) - - @contextmanager - def _state(self) -> Iterator[dict]: - if self.state_file is None: - yield self._local_state - return - parent = self.state_file.parent - parent.mkdir(mode=0o700, parents=True, exist_ok=True) - owner = parent.stat() - descriptor = os.open(str(self.state_file) + ".lock", os.O_CREAT | os.O_RDWR, 0o600) - with os.fdopen(descriptor, "r+") as lock: - # rjob worker 以 root 运行时,私有状态仍归目录所有者管理。 - if os.geteuid() == 0: - os.fchown(lock.fileno(), owner.st_uid, owner.st_gid) - fcntl.flock(lock, fcntl.LOCK_EX) - state = json.loads(self.state_file.read_text()) if self.state_file.exists() else {} - yield state.setdefault(self.provider, {}) - descriptor, temporary = tempfile.mkstemp(prefix=self.state_file.name + ".", dir=parent) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - if os.geteuid() == 0: - os.fchown(stream.fileno(), owner.st_uid, owner.st_gid) - json.dump(state, stream, ensure_ascii=False) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, self.state_file) - finally: - Path(temporary).unlink(missing_ok=True) - - def _available(self, state: dict, excluded: set[str] | None = None) -> list[dict]: - excluded = excluded or set() - now = time.time() - cooldowns = state.get("cooldowns", {}) - return [row for row in self.keys if row["id"] not in excluded - and cooldowns.get(row["id"], {}).get("until", 0) <= now] - - def select(self, excluded: set[str] | None = None) -> dict: - with self._state() as state: - cooldowns = state.get("cooldowns", {}) - available = self._available(state, excluded) - if not available: - next_retry = min( - (cooldowns.get(row["id"], {}).get("until", 0) for row in self.keys), default=0 - ) - raise PoolExhausted( - f"pool_exhausted: provider={self.provider}, next_retry_at={next_retry}" - ) - if self._preferred_id is not None: - preferred = next((row for row in available if row["id"] == self._preferred_id), None) - if preferred is not None: - return preferred - tier = self.keys - cursor_name = "selected:" + self._selection_group - else: - priority = min(row["priority"] for row in available) - tier = [row for row in self.keys if row["priority"] == priority] - cursor_name = str(priority) - cursors = state.setdefault("cursors", {}) - cursor = cursors.get(cursor_name, 0) - indices = ((cursor + offset) % len(tier) for offset in range(len(tier))) - index = next(index for index in indices if tier[index] in available) - cursors[cursor_name] = (index + 1) % len(tier) - return tier[index] - - def cooldown(self, key_id: str, seconds: float, reason: str) -> None: - with self._state() as state: - cooldowns = state.setdefault("cooldowns", {}) - previous = cooldowns.get(key_id, {}).get("until", 0) - cooldowns[key_id] = {"until": max(previous, time.time() + seconds), "reason": reason} - - def redact(self, value): - if isinstance(value, str): - for secret in self._secrets: - value = value.replace(secret, "[REDACTED]") - return value - if isinstance(value, dict): - return {key: self.redact(item) for key, item in value.items()} - if isinstance(value, list): - return [self.redact(item) for item in value] - return value diff --git a/src/tandem/models/slow.py b/src/tandem/models/slow.py index 0d569a3..159f60f 100644 --- a/src/tandem/models/slow.py +++ b/src/tandem/models/slow.py @@ -12,11 +12,12 @@ import time import urllib.error import urllib.request +from email.utils import parsedate_to_datetime from pathlib import Path from PIL import Image -from .key_pool import ApiKeyPool, PoolExhausted, classify_api_error, retry_after_seconds +from .api_errors import classify_api_error # 空正文(全留在思考通道)或截断时,下一次重试升档思考强度,避免原样重发重复失败 _REASON_EFFORT_LADDER = {"low": "medium", "medium": "high"} @@ -131,13 +132,25 @@ def _api_failure(status: int, raw: dict): return base.get("status_code"), str(error or base.get("status_msg") or json.dumps(raw)) +def _retry_after_seconds(value: str | None, default: float) -> float: + """Honor the endpoint's Retry-After without maintaining account state.""" + if value is None: + return default + try: + return max(0.0, float(value)) + except ValueError: + try: + return max(0.0, parsedate_to_datetime(value).timestamp() - time.time()) + except (TypeError, ValueError): + return default + + class SlowModelClient: def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): if isinstance(credential_file, dict): self.credentials = dict(credential_file) else: self.credentials = json.loads(Path(credential_file).read_text(encoding="utf-8")) - self.key_pool = ApiKeyPool(self.credentials) self._call_id = 0 self.config = config self.audit_path = audit_path @@ -146,6 +159,16 @@ def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): self.llm_sink = llm_sink self.deadline = None + def _redact(self, value): + if isinstance(value, str): + secret = self.credentials.get("api_key") + return value.replace(secret, "[REDACTED]") if secret else value + if isinstance(value, dict): + return {key: self._redact(item) for key, item in value.items()} + if isinstance(value, list): + return [self._redact(item) for item in value] + return value + def set_deadline(self, deadline): """Absolute monotonic task deadline, shared by requirements, planning and verification.""" self.deadline = deadline @@ -200,8 +223,8 @@ def build_request(): data=data, headers={ **( - {"Authorization": "Bearer " + key["api_key"]} - if key["api_key"] + {"Authorization": "Bearer " + credentials.get("api_key", "")} + if credentials.get("api_key", "") else {} ), "Content-Type": "application/json", @@ -218,11 +241,11 @@ def audit(event, started, attempt, **details): record = { "event": event, "provider": "slow_model", - "api_provider": self.key_pool.provider, + "api_provider": "custom", "role": role, "call_id": call_id, - "key_id": key["id"], - "key_priority": key["priority"], + "key_id": "manual", + "key_priority": 1, "attempt": attempt, "time": time.time(), "elapsed_seconds": round(time.perf_counter() - started, 3), @@ -230,7 +253,7 @@ def audit(event, started, attempt, **details): **image_stats, **details, } - text = json.dumps(self.key_pool.redact(record), ensure_ascii=False) + text = json.dumps(self._redact(record), ensure_ascii=False) print(text, flush=True) with self.audit_path.open("a", encoding="utf-8") as log: log.write(text + "\n") @@ -250,19 +273,6 @@ def public_headers(headers): } return {name.lower(): value for name, value in headers.items() if name.lower() in allowed} - excluded: set[str] = set() - - def alternate_key(category, seconds, started, attempt): - if len(self.key_pool.keys) == 1: - return key # A manually configured endpoint keeps its bounded retry path. - self.key_pool.cooldown(key["id"], seconds, category) - excluded.add(key["id"]) - audit("key_cooldown", started, attempt, category=category, cooldown_seconds=seconds) - replacement = self.key_pool.select(excluded) - audit("key_changed", started, attempt, category=category, next_key_id=replacement["id"]) - return replacement - - key = self.key_pool.select() attempt = 0 failures = 0 while True: @@ -280,7 +290,7 @@ def alternate_key(category, seconds, started, attempt): except urllib.error.HTTPError as error: status = error.code headers = public_headers(error.headers) - body = self.key_pool.redact(error.read().decode("utf-8", errors="replace")) + body = self._redact(error.read().decode("utf-8", errors="replace")) try: raw = json.loads(body) except ValueError: @@ -290,40 +300,28 @@ def alternate_key(category, seconds, started, attempt): if self.llm_sink is not None: self.llm_sink( role=role, - request=self.key_pool.redact({"payload": payload, "messages": messages}), - response=self.key_pool.redact(raw), + request=self._redact({"payload": payload, "messages": messages}), + response=self._redact(raw), attempt=attempt, call_id=call_id, - key_id=key["id"], key_priority=key["priority"], + key_id="manual", key_priority=1, http_status=status, elapsed_seconds=round(time.perf_counter() - started, 3), ) failure = _api_failure(status, raw) if failure is not None: code, message = failure - category, cooldown = classify_api_error(self.key_pool.provider, status, code, message) + category = classify_api_error(status, raw) audit("api_error", started, attempt, http_status=status, headers=headers, error_code=code, category=category, error=message[:2000], usage=raw.get("usage")) - error = RuntimeError(self.key_pool.redact( + error = RuntimeError(self._redact( f"slow model API error: status={status}, code={code}, message={message[:2000]}" )) - if cooldown: - cooldown = retry_after_seconds(headers.get("retry-after"), cooldown) - self.key_pool.cooldown(key["id"], cooldown, category) - excluded.add(key["id"]) - audit("key_cooldown", started, attempt, category=category, - cooldown_seconds=cooldown) - try: - key = self.key_pool.select(excluded) - except PoolExhausted as exhausted: - raise exhausted from error - continue if category != "transient": raise error failures += 1 if failures >= 3: raise error - delay = retry_after_seconds(headers.get("retry-after"), 30) - key = alternate_key("transient", max(60, delay), started, attempt) + delay = _retry_after_seconds(headers.get("retry-after"), 30) audit("retry_wait", started, attempt, retry_wait_seconds=delay) self._retry_wait(delay) continue @@ -362,7 +360,7 @@ def alternate_key(category, seconds, started, attempt): started, attempt, error_type=type(error).__name__, - error=self.key_pool.redact(str(error))[:1000], + error=self._redact(str(error))[:1000], ) failures += 1 if failures >= 3: @@ -373,7 +371,6 @@ def alternate_key(category, seconds, started, attempt): http.client.IncompleteRead)): if isinstance(error, TaskBudgetExceeded): raise - key = alternate_key("transport_error", 60, started, attempt) elif (self.config.execution_mode == "goal" and role == "plan_stage" and "finish_reason=length" in str(error)): payload["reasoning_effort"] = "low" diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index 8dad50e..0947fc6 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -41,8 +41,6 @@ def main(argv: list[str] | None = None) -> int: help="阶段内动作数上限,达到后请求剩余工作的重规划") run_cmd.add_argument("--slow-model", default=None, metavar="NAME", help="慢模型(大脑)API 名:config 的 harness.slow_model.apis 里选;默认取 default") - run_cmd.add_argument("--slow-key-ids", default=None, metavar="1,2,3", - help="号池 key 编号;多个编号按 worker 均分,未指定则取配置") run_cmd.add_argument("--fast-model", default=None, metavar="NAME", help="快模型(小脑)API 名:config 的 harness.fast_model.apis 里选;默认取 default") run_cmd.add_argument("--slow-max-tokens", type=int, default=None, @@ -78,14 +76,6 @@ def main(argv: list[str] | None = None) -> int: config = load_config(args.config) if args.rjob_command == "run": overrides: dict = {"benchmark": args.benchmark} - if args.slow_key_ids is not None: - try: - overrides["slow_key_ids"] = ( - [int(item.strip()) for item in args.slow_key_ids.split(",")] - if args.slow_key_ids.strip() else [] - ) - except ValueError: - parser.error("--slow-key-ids 需填写正整数编号,多个编号用逗号分隔") manifest: dict = {} if args.tasks: manifest["tasks"] = [item.strip() for item in args.tasks.split(",") if item.strip()] diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index 6d696c1..cef65a3 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -20,8 +20,6 @@ from pathlib import Path from typing import Any -from tandem.models.key_pool import POOL_BASE_URLS, validate_key_ids - #: 必填路径(点号路径,逐段下钻;None / 空串 / 空列表视为缺失) REQUIRED_PATHS = ( "harness.slow_model.default", @@ -123,28 +121,8 @@ def _validate_slow_models(config: dict) -> list[str]: base_url = api.get("base_url") if isinstance(base_url, str) and base_url.strip() and not base_url.startswith("http"): problems.append(f"harness.slow_model.apis.{name}.base_url(应为 http(s) 地址)") - pool = api.get("key_pool") - if isinstance(pool, dict): - try: - validate_key_ids(pool.get("key_ids", [])) - except ValueError: - problems.append(f"harness.slow_model.apis.{name}.key_pool.key_ids(应为不重复的正整数列表)") - if api.get("api_key") and pool.get("key_ids"): - problems.append(f"harness.slow_model.apis.{name}(api_key 与 key_ids 不能同时指定)") - if _is_missing(api.get("api_key")): - prefix = f"harness.slow_model.apis.{name}.key_pool" - if not isinstance(pool, dict): - problems.append(f"harness.slow_model.apis.{name}.api_key / .key_pool(至少给一个)") - else: - provider = pool.get("provider") - if provider not in POOL_BASE_URLS: - problems.append(f"{prefix}.provider(应为 minimax / kimi)") - elif not isinstance(base_url, str) or base_url.rstrip("/") != POOL_BASE_URLS[provider]: - problems.append(f"{prefix}.provider 与官方 base_url 不匹配") - for key in ("credentials_file", "state_file"): - path = pool.get(key) - if not isinstance(path, str) or not path.startswith("/"): - problems.append(f"{prefix}.{key}(必须是绝对路径)") + if not isinstance(api.get("api_key", ""), str): + problems.append(f"harness.slow_model.apis.{name}.api_key(应为字符串)") # 思考强度必须落在 harness 认识的档位里,否则 SessionConfig 会在运行期才报错 effort = api.get("reasoning_effort") if effort is not None and effort not in REASONING_EFFORTS: diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 0e8a1fc..c9fff1d 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -20,8 +20,6 @@ import subprocess from pathlib import Path -from tandem.models.key_pool import validate_key_ids - from .config_schema import select_api @@ -79,8 +77,6 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic }, "registry": config["eval"].get("registry"), } - if not slow_api.get("api_key") and slow_api.get("key_pool"): - secrets["slow_model"]["key_pool"] = dict(slow_api["key_pool"]) if benchmark == "osworld21": section = config["eval"]["benchmarks"][benchmark] environment = {} @@ -167,14 +163,6 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, requested = dict(overrides.pop("models", {}) or {}) slow_name, slow_api = select_api(config, "slow_model", requested.get("slow")) fast_name, fast_api = select_api(config, "fast_model", requested.get("fast")) - key_ids = overrides.pop("slow_key_ids", None) - if key_ids is not None: - validate_key_ids(key_ids) - if key_ids and (slow_api.get("api_key") or not slow_api.get("key_pool")): - raise ValueError("slow_key_ids requires key_pool without a manual api_key") - if slow_api.get("key_pool"): - slow_api["key_pool"] = {**slow_api["key_pool"], "key_ids": key_ids} - inner_image = config["eval"]["inner_image"] if benchmark == "osworld21": # OSW2.1 用发布清单固定的运行镜像(digest),不覆盖 latest,也不复用 OSW2 的镜像 diff --git a/tests/eval/test_experiment_script.py b/tests/eval/test_experiment_script.py index 9773b32..a7cdd9c 100644 --- a/tests/eval/test_experiment_script.py +++ b/tests/eval/test_experiment_script.py @@ -17,7 +17,6 @@ def test_dry_run_creates_manifest_and_batch_log(tmp_path): "--tasks", "024,029", "--concurrency", "12", "--slow-model", "kimi-k3", - "--slow-key-ids", "1,2,3,4", "--max-steps", "500", "--max-wall", "14400", "--dry-run", @@ -33,7 +32,6 @@ def test_dry_run_creates_manifest_and_batch_log(tmp_path): assert manifest["tasks"] == ["024", "029"] assert manifest["concurrency"] == 12 assert manifest["slow_model"] == "kimi-k3" - assert manifest["slow_key_ids"] == [1, 2, 3, 4] assert manifest["max_steps"] == 500 assert manifest["max_wall_seconds"] == 14400.0 assert manifest["watch_timeout_seconds"] == 15000.0 diff --git a/tests/harness/test_slow_api_errors.py b/tests/harness/test_slow_api_errors.py new file mode 100644 index 0000000..56bcbba --- /dev/null +++ b/tests/harness/test_slow_api_errors.py @@ -0,0 +1,88 @@ +"""Provider response errors retain retry semantics without account state.""" + +import io +import json +import urllib.request + +import pytest + +from tandem.config import SessionConfig +from tandem.models.slow import SlowModelClient + + +class _Response(io.BytesIO): + status = 200 + headers = {} + + +@pytest.fixture +def client(tmp_path): + return SlowModelClient({ + "base_url": "https://api.minimax.cn/v1/", "api_key": "test-only-key", + "model": "MiniMax-M3.1-Flash", + }, SessionConfig(), tmp_path / "audit.jsonl") + + +@pytest.mark.parametrize("base_url", ["https://api.minimax.cn/v1/", "http://gateway/v1/"]) +@pytest.mark.parametrize("code_type", [int, str]) +@pytest.mark.parametrize("code", [1000, 1001, 1024, 1033, 2064, 2066]) +def test_minimax_http_200_temporary_error_recovers_with_same_request( + client, monkeypatch, base_url, code_type, code): + client.credentials["base_url"] = base_url + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + raw = ({"base_resp": {"status_code": code_type(code), "status_msg": "temporary failure"}} + if len(requests) == 1 else + {"choices": [{"message": {"content": '{"ok":true}'}, "finish_reason": "stop"}]}) + return _Response(json.dumps(raw).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + assert client.json_request([{"role": "user", "content": "probe"}], lambda v: v == {"ok": True}, + role="plan_stage") == {"ok": True} + assert len(requests) == 2 and waits == [30] + assert requests[0].data == requests[1].data + assert all(r.full_url == base_url.rstrip("/") + "/chat/completions" for r in requests) + assert all(r.get_header("Authorization") == "Bearer test-only-key" for r in requests) + assert sorted(p.name for p in client.audit_path.parent.iterdir()) == ["audit.jsonl"] + + +@pytest.mark.parametrize("code", [1000, 1001, 1024, 1033, 2064, 2066]) +def test_minimax_temporary_error_retries_remain_bounded(client, monkeypatch, code): + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + return _Response(json.dumps({"base_resp": {"status_code": code, "status_msg": "busy"}}).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError, match=f"code={code}"): + client.json_request([{"role": "user", "content": "probe"}], lambda v: True, role="plan_stage") + assert len(requests) == 3 and waits == [30, 30] + + +@pytest.mark.parametrize("status,raw", [ + (200, {"base_resp": {"status_code": 1004, "status_msg": "invalid key"}}), + (200, {"base_resp": {"status_code": 1008, "status_msg": "quota exhausted"}}), + (200, {"base_resp": {"status_code": 9999, "status_msg": "unknown failure"}}), + (200, {"error": {"code": 1000, "message": "unrelated provider error"}}), + (400, {"base_resp": {"status_code": 1000, "status_msg": "bad request"}}), + (401, {"base_resp": {"status_code": 1000, "status_msg": "unauthorized"}}), +]) +def test_other_api_errors_do_not_gain_retries(client, monkeypatch, status, raw): + requests, waits = [], [] + + def opener(request, **kwargs): + requests.append(request) + response = _Response(json.dumps(raw).encode()) + response.status = status + return response + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError): + client.json_request([{"role": "user", "content": "probe"}], lambda v: True, role="plan_stage") + assert len(requests) == 1 and waits == [] diff --git a/tests/harness/test_slow_transport.py b/tests/harness/test_slow_transport.py index 753dc26..957ddd6 100644 --- a/tests/harness/test_slow_transport.py +++ b/tests/harness/test_slow_transport.py @@ -200,25 +200,19 @@ def opener(request, **kwargs): @pytest.fixture -def pooled_client(tmp_path, monkeypatch): - credentials = tmp_path / "synthetic-credentials.json" - credentials.write_text(json.dumps({"kimi": [ - {"id": f"synthetic-{i}", "priority": 1, "api_key": f"test-only-{i}"} - for i in range(3) - ]})) +def endpoint_client(tmp_path, monkeypatch): monkeypatch.setattr("tandem.models.slow.time.sleep", lambda seconds: None) return SlowModelClient({ - "base_url": "https://api.kimi.com/coding/v1", "model": "synthetic", - "key_pool": {"provider": "kimi", "key_ids": [1, 2, 3], - "credentials_file": str(credentials), "state_file": str(tmp_path / "state.json")}, + "base_url": "http://gateway/v1/", "model": "MiniMax-M3.1-Flash", + "api_key": "test-only-gateway-key", }, SessionConfig(slow_reasoning_effort="low"), tmp_path / "audit.jsonl") @pytest.mark.parametrize("role", ["plan_stage", "verify_stage", "ground_visual"]) @pytest.mark.parametrize("failure", [TimeoutError, http.client.RemoteDisconnected, urllib.error.URLError, http.client.IncompleteRead]) -def test_transport_failure_switches_key_without_changing_reasoning_or_replaying_actions( - pooled_client, monkeypatch, role, failure): +def test_transport_failure_retries_endpoint_without_changing_reasoning_or_replaying_actions( + endpoint_client, monkeypatch, role, failure): requests = [] def opener(request, **kwargs): @@ -230,18 +224,21 @@ def opener(request, **kwargs): return _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role=role) == {"done": True} - assert [r.get_header("Authorization") for r in requests] == ["Bearer test-only-0", "Bearer test-only-1"] + assert len(requests) == 2 + assert all(r.full_url == "http://gateway/v1/chat/completions" for r in requests) + assert all(r.get_header("Authorization") == "Bearer test-only-gateway-key" for r in requests) assert requests[0].data == requests[1].data - events = [json.loads(line) for line in pooled_client.audit_path.read_text().splitlines()] - changed = next(e for e in events if e["event"] == "key_changed") - assert (changed["key_id"], changed["next_key_id"]) == ("synthetic-0", "synthetic-1") - assert "test-only-" not in pooled_client.audit_path.read_text() - assert pooled_client.key_pool.select()["id"] != "synthetic-0" + payload = json.loads(requests[0].data) + assert payload["model"] == "MiniMax-M3.1-Flash" and payload["stream"] is False + events = [json.loads(line) for line in endpoint_client.audit_path.read_text().splitlines()] + assert not any(e["event"] in {"key_cooldown", "key_changed"} for e in events) + assert "test-only-gateway-key" not in endpoint_client.audit_path.read_text() + assert sorted(p.name for p in endpoint_client.audit_path.parent.iterdir()) == ["audit.jsonl"] -def test_temporary_server_error_switches_key(pooled_client, monkeypatch): +def test_temporary_server_error_retries_same_endpoint(endpoint_client, monkeypatch): requests = [] class Unavailable(_Response): @@ -255,13 +252,14 @@ def opener(request, **kwargs): return Unavailable() if len(requests) == 1 else _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") == {"done": True} - assert requests[0].get_header("Authorization") != requests[1].get_header("Authorization") + assert len(requests) == 2 + assert requests[0].get_header("Authorization") == requests[1].get_header("Authorization") @pytest.mark.parametrize("failure", [TimeoutError, http.client.IncompleteRead]) -def test_transport_key_rotation_is_bounded(pooled_client, monkeypatch, failure): +def test_endpoint_transport_retries_are_bounded(endpoint_client, monkeypatch, failure): requests = [] def opener(request, **kwargs): @@ -272,12 +270,13 @@ def opener(request, **kwargs): monkeypatch.setattr(urllib.request, "urlopen", opener) with pytest.raises(failure): - pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") - assert len(requests) == len({r.get_header("Authorization") for r in requests}) == 3 + assert len(requests) == 3 + assert len({r.get_header("Authorization") for r in requests}) == 1 -def test_invalid_json_does_not_cool_down_or_switch_key(pooled_client, monkeypatch): +def test_invalid_json_repair_uses_same_endpoint(endpoint_client, monkeypatch): requests = [] class Invalid(_Response): @@ -289,7 +288,66 @@ def opener(request, **kwargs): return Invalid() if len(requests) == 1 else _Response() monkeypatch.setattr(urllib.request, "urlopen", opener) - assert pooled_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, role="plan_stage") == {"done": True} assert requests[0].get_header("Authorization") == requests[1].get_header("Authorization") - assert "key_cooldown" not in pooled_client.audit_path.read_text() + assert "key_cooldown" not in endpoint_client.audit_path.read_text() + + +@pytest.mark.parametrize("status", [400, 401, 402, 403]) +def test_gateway_errors_are_redacted_without_local_account_state(endpoint_client, monkeypatch, status): + calls, captured = [], [] + secret = endpoint_client.credentials["api_key"] + endpoint_client.llm_sink = lambda **record: captured.append(record) + + def opener(request, **kwargs): + calls.append(request) + body = json.dumps({"error": {"message": "gateway rejected " + secret}}).encode() + raise urllib.error.HTTPError(request.full_url, status, "gateway error", {}, io.BytesIO(body)) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + with pytest.raises(RuntimeError, match=f"status={status}") as raised: + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") + assert len(calls) == 1 and captured[0]["http_status"] == status + assert secret not in str(raised.value) + endpoint_client.audit_path.read_text() + json.dumps(captured) + assert "[REDACTED]" in json.dumps(captured) + assert sorted(p.name for p in endpoint_client.audit_path.parent.iterdir()) == ["audit.jsonl"] + + +@pytest.mark.parametrize("status", [408, 429, 503]) +@pytest.mark.parametrize("headers,delay", [({}, 30), ({"Retry-After": "2"}, 2)]) +def test_gateway_retries_honor_retry_after_and_remain_bounded( + endpoint_client, monkeypatch, status, headers, delay): + calls, waits = [], [] + + def opener(request, **kwargs): + calls.append(request) + raise urllib.error.HTTPError(request.full_url, status, "busy", headers, + io.BytesIO(b'{"error":{"message":"busy"}}')) + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + with pytest.raises(RuntimeError, match=f"status={status}"): + endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") + assert len(calls) == 3 and waits == [delay, delay] + assert len({r.get_header("Authorization") for r in calls}) == 1 + + +def test_gateway_429_can_recover_on_last_attempt(endpoint_client, monkeypatch): + calls, waits = [], [] + + def opener(request, **kwargs): + calls.append(request) + if len(calls) < 3: + raise urllib.error.HTTPError(request.full_url, 429, "busy", {}, + io.BytesIO(b'{"error":{"message":"rate limited"}}')) + return _Response() + + monkeypatch.setattr(urllib.request, "urlopen", opener) + monkeypatch.setattr("tandem.models.slow.time.sleep", waits.append) + assert endpoint_client.json_request([{"role": "user", "content": "plan"}], lambda v: True, + role="plan_stage") == {"done": True} + assert len(calls) == 3 and waits == [30, 30] + assert len({r.data for r in calls}) == 1