From 9a20676dcf545040782f04c166205d0043345753 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Fri, 28 Aug 2026 18:39:23 +0800 Subject: [PATCH 01/12] feat: add phase 2 observability and audit retention --- .env.example | 4 + CHANGELOG.md | 11 + README.md | 36 +- ...20260828_0005_worker_progress_retention.py | 94 +++ backend/app/api/v1/health.py | 471 +++--------- backend/app/api/v1/observability.py | 242 +++++++ backend/app/api/v1/router.py | 2 + backend/app/api/v1/runs.py | 108 +-- backend/app/cli/audit_retention.py | 159 +++++ backend/app/core/config.py | 15 + backend/app/core/logging.py | 119 +++- backend/app/core/logging_contract.py | 472 ++++++++++++ backend/app/db/__init__.py | 32 +- backend/app/db/import_sqlite.py | 144 +++- backend/app/db/prepare_migrations.py | 60 +- backend/app/governance/__init__.py | 90 ++- backend/app/governance/audit.py | 641 +++++++++++++---- backend/app/governance/audit_archive.py | 669 ++++++++++++++++++ backend/app/governance/audit_retention.py | 469 ++++++++++++ .../app/governance/audit_retention_errors.py | 35 + backend/app/main.py | 17 +- backend/app/models/__init__.py | 2 + backend/app/models/governance.py | 2 + backend/app/models/worker_process.py | 57 ++ backend/app/observability/__init__.py | 37 + backend/app/observability/prometheus.py | 362 ++++++++++ backend/app/observability/snapshot.py | 542 ++++++++++++++ backend/app/runners/evaluation_runner.py | 33 + backend/app/schemas/system.py | 11 + backend/app/task_queue/redis_streams.py | 29 +- backend/app/worker.py | 26 +- backend/app/worker_probe.py | 15 +- backend/app/worker_progress.py | 354 +++++++++ backend/app/workers/service.py | 22 + backend/pyproject.toml | 1 + .../fixtures/audit-archive-v1-empty.jsonl | 2 + .../fixtures/audit-archive-v1-settled.jsonl | 3 + .../test_postgres_audit_retention.py | 242 +++++++ .../tests/integration/test_postgres_leases.py | 53 +- .../tests/integration/test_redis_streams.py | 10 +- backend/tests/test_api.py | 25 + backend/tests/test_audit_api.py | 130 ++++ backend/tests/test_audit_archive.py | 316 +++++++++ backend/tests/test_audit_retention.py | 335 +++++++++ backend/tests/test_audit_retention_cli.py | 207 ++++++ .../test_evaluation_runner_reliability.py | 33 + backend/tests/test_logging.py | 287 +++++++- backend/tests/test_logging_sources.py | 134 ++++ backend/tests/test_migrations.py | 114 ++- .../tests/test_phase2_acceptance_script.py | 175 +++++ backend/tests/test_phase2_capacity_script.py | 24 +- backend/tests/test_prometheus_alert_rules.py | 137 ++++ backend/tests/test_prometheus_exporter.py | 660 +++++++++++++++++ backend/tests/test_sqlite_postgres_import.py | 214 +++++- backend/tests/test_task_queue.py | 58 +- backend/tests/test_worker.py | 62 +- backend/tests/test_worker_probe.py | 11 +- backend/tests/test_worker_progress.py | 240 +++++++ compose.yaml | 4 + deploy/observability/prometheus-alerts.json | 130 ++++ .../prometheus-scrape.example.yml | 7 + docs/API.md | 32 +- docs/ARCHITECTURE.md | 24 +- docs/DEPLOYMENT.md | 53 +- docs/NEXT_TASK.md | 143 ++-- docs/OPERATIONS.md | 141 +++- docs/PROJECT_STATUS.md | 37 +- docs/ROADMAP.md | 23 +- docs/SECURITY.md | 27 +- docs/TESTING.md | 57 +- ...ability-worker-progress-audit-retention.md | 330 +++++++++ docs/phases/PHASE-2-RELIABILITY.md | 38 +- ...6-08-28-phase-2-observability-retention.md | 141 ++++ ...6-08-28-phase-2-observability-retention.md | 158 +++++ scripts/phase2_acceptance.py | 447 ++++++++++-- scripts/phase2_capacity.py | 1 + 76 files changed, 9655 insertions(+), 993 deletions(-) create mode 100644 backend/alembic/versions/20260828_0005_worker_progress_retention.py create mode 100644 backend/app/api/v1/observability.py create mode 100644 backend/app/cli/audit_retention.py create mode 100644 backend/app/core/logging_contract.py create mode 100644 backend/app/governance/audit_archive.py create mode 100644 backend/app/governance/audit_retention.py create mode 100644 backend/app/governance/audit_retention_errors.py create mode 100644 backend/app/models/worker_process.py create mode 100644 backend/app/observability/__init__.py create mode 100644 backend/app/observability/prometheus.py create mode 100644 backend/app/observability/snapshot.py create mode 100644 backend/app/worker_progress.py create mode 100644 backend/tests/fixtures/audit-archive-v1-empty.jsonl create mode 100644 backend/tests/fixtures/audit-archive-v1-settled.jsonl create mode 100644 backend/tests/integration/test_postgres_audit_retention.py create mode 100644 backend/tests/test_audit_archive.py create mode 100644 backend/tests/test_audit_retention.py create mode 100644 backend/tests/test_audit_retention_cli.py create mode 100644 backend/tests/test_logging_sources.py create mode 100644 backend/tests/test_prometheus_alert_rules.py create mode 100644 backend/tests/test_prometheus_exporter.py create mode 100644 backend/tests/test_worker_progress.py create mode 100644 deploy/observability/prometheus-alerts.json create mode 100644 deploy/observability/prometheus-scrape.example.yml create mode 100644 docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md create mode 100644 docs/plans/2026-08-28-phase-2-observability-retention.md create mode 100644 docs/worklogs/2026-08-28-phase-2-observability-retention.md diff --git a/.env.example b/.env.example index 1b269f1..0972348 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,10 @@ LLMBENCHLAB_WORKER_HEARTBEAT_SECONDS=10 LLMBENCHLAB_WORKER_POLL_SECONDS=1 LLMBENCHLAB_WORKER_MAX_ATTEMPTS=3 LLMBENCHLAB_WORKER_SHUTDOWN_GRACE_SECONDS=30 +LLMBENCHLAB_WORKER_PROGRESS_FLUSH_SECONDS=5 +LLMBENCHLAB_WORKER_PROGRESS_STALE_SECONDS=60 +LLMBENCHLAB_WORKER_EXPECTED_PROCESSES=1 +LLMBENCHLAB_WORKER_RECOVERY_ALERT_SECONDS=60 LLMBENCHLAB_REDIS_PUBLISH_TIMEOUT_SECONDS=1 LLMBENCHLAB_REDIS_OPERATION_TIMEOUT_SECONDS=2 diff --git a/CHANGELOG.md b/CHANGELOG.md index d9558db..cf635b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Added +- Added the in-progress P2-06 implementation under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. +- Added Alembic revision `20260828_0005`, the `worker_processes` generation table, bounded `(expires_at,id)` / `(occurred_at,id)` audit scan indexes, and DB-time coalesced Worker scan/claim/lease-heartbeat/progress recording. `/tasks/metrics` and the exporter expose only low-cardinality aggregates; the dependency probe remains capability-only. The stopped SQLite-to-PostgreSQL importer now copies and reconciles all 13 application tables, rejects a still-live Worker generation, and preserves stopped/stale process facts. +- Added `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete`. The v1 canonical JSONL archive is bounded, exact-digest bound, atomically installed without replacement at `0600`, and independently verifiable without a database engine; delete is never implicit, restore/delete require an exact confirmed archive SHA-256, and commit-unknown outcomes require read-only reconciliation before retry. These integrity hashes are neither signatures nor WORM guarantees. - Added the Phase 2 single-host control-plane qualification under ADR-0012 through ADR-0014. The `P2-local-control-plane-v2` harness fixes one warm-up plus exactly five measured trials, four Mock-only capacity cells, paired Worker scaling, queue/execution/end-to-end and recovery objectives, ledger-derived correctness gates, bounded capacity formulas, and scoped project-image cleanup. Clean implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` passed 23/23 SLO checks with all six child invariants and cleanup checks; the earlier v1 aggregate remains permanently `unqualified` rather than being discarded or reclassified. This qualifies only the recorded single-host Mock control-plane profile, not production SLA, real-Provider capacity, HA, billing, or external exactly-once behavior. - Added the current Phase 2 governance/audit slice under ADR-0009, as amended by ADR-0010 and ADR-0011: database-authoritative global/provider/model/run admission, immutable policy/hash plus per-Run override snapshots, per-Provider-attempt reservation/settlement ledger, fixed-window RPM/TPM and lifetime request/Token/cost limits, bounded backlog, cooperative question quantum, typed audit/history APIs, and non-secret Provider evidence. - Added Alembic revision `20260827_0004` with six governance/audit tables (`governance_policies`, `governance_scopes`, `governance_minute_buckets`, `question_executions`, `provider_call_reservations`, and `audit_events`) plus Run/Response operational evidence. The stopped SQLite-to-PostgreSQL importer now copies and reconciles all 12 application tables in dependency order. @@ -36,6 +39,9 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Changed +- Governed production logging at the source boundary: application calls use literal argument-free messages, structured values remain allowlisted and finite, third-party logger text/identity is replaced by fixed classification, and the raw Uvicorn access handler is disabled rather than bypassing the JSON redaction contract. +- Extended the Phase 2 migration/importer head from `20260827_0004` / 12 tables to `20260828_0005` / 13 tables. A populated `worker_processes` table blocks `0005 -> 0004` before DDL; only an explicitly emptied table may downgrade, after which the existing governance/audit `0004` guards still apply. +- Hardened the final P2-06 integrity boundaries: importer reconciliation now compares committed target facts through the same canonical contract, no-op retention mutations still execute post-commit verification, PostgreSQL archive/delete/restore operations hold advisory plus row locks, and audit archives enforce the record-count cap before per-line decoding. - OpenAI-compatible Chat Completions now explicitly requests `stream:true` plus streamed usage, consumes token/heartbeat/usage SSE events through `[DONE]`, and retains a normal-JSON success fallback. `read_timeout_seconds` is documented as the idle wait for the next response bytes rather than a total generation wall-clock limit. - Standardized `score`, `completion_rate`, and `answered_accuracy` as 0–100 values across implementation, API documentation, protocol, and ADR. - Configured local CORS for both `localhost:5173` and `127.0.0.1:5173`, while continuing to reject wildcard origins. @@ -57,6 +63,8 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Fixed +- Prevented external `LogRecord` extras from injecting allowlisted request/run fields, rejected FIFO/non-regular archive inputs without blocking by opening them nonblocking before `fstat`, and made oversized audit archives fail on their global line cap before decoding attacker-controlled lines. +- Made importer post-commit verification and no-op retention post-verification fail closed instead of treating an unchanged row count as sufficient evidence; PostgreSQL retention paths now preserve their advisory/row-lock gate through the mutation decision. - Made materialized governance scope/minute counters fail closed when they diverge from the never-delete attempt ledger; low or high counter drift cannot be used to bypass admission, mutation, reconciliation, or importer preflight. - Prevented a confirmed `released_pre_send` reservation from consuming a Provider retry ordinal. The old ledger row remains terminal and unique while the next local generation resumes at the first unsent HTTP ordinal, including `max_retries=0`. - Removed a CI-only race from the Runs-page polling tests by waiting for and selecting the page's 2-second interval instead of assuming the most recent global timer belongs to the component; Testing Library may concurrently own a 50-millisecond wait timer. @@ -77,6 +85,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Security +- Restricted the new exporter to fixed metric families and enum labels with no Run/Model/Provider/Worker/question/reservation IDs, URLs, hashes, origins, correlation IDs, or error text. Audit archives use a strict full-row allowlist and never admit API keys, Authorization/Cookie values, credential ciphertext/nonces/keyring, Provider URLs, prompts, questions, Responses, or raw Provider bodies; archive paths and invalid input are not reflected by CLI errors. - Restricted governance and credential audit payloads to typed allowlists and stable opaque identifiers; no API Key, Authorization header, credential ciphertext/nonce, keyring material, Provider URL, question text, prompt, or response body is admitted. Audit/policy/counter corruption fails closed and records only a fixed integrity event in an independent transaction where possible. - Bounded streaming Chat responses independently at 64 MiB of SSE wire data, 1 MiB per event, and 4 MiB of aggregated content while preserving the 4 MiB JSON/64 KiB error limits. Raw SSE lines are not logged, and content is aggregated before exact current-Key replacement so a Key split across deltas is still removed. - Restricted Benchmark ZIP import by size, entry name/type, compression ratio, schema, and fixed root filenames; dataset contents are never executed. @@ -98,6 +107,8 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Verification +- P2-06 combined targeted regression suites are green. On the current uncommitted worktree, `make lint` passed Ruff across 152 files plus ESLint and TypeScript typecheck; `make test` passed with `916 passed, 33 skipped` on the backend and `38 passed` on the frontend; offline Mock smoke passed `1 passed, 7 deselected`; a temporary PostgreSQL 16/Redis 7 environment passed migration/check and `33 passed, 0 skipped` integration; temporary SQLite head-to-`0001`-to-head migration/check, the frontend production build, Compose config, and Prometheus `v3.5.0` `promtool check rules` for all eight rules are green. Dirty-tree Compose acceptance passed 9/9 with evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`), Worker gauges `2/2/2/0/0`, both populated downgrade refusals, two empty-database round-trips, and empty container/volume/network cleanup. The latest dirty capacity also passed with evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`): 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit events, zero error/drift/duplicate/PEL/lag, expected Workers 2, and empty container/volume/network/image cleanup; it is an offline Mock observation, not an SLO. Code review reported 0 Blocker/High/Medium; the later staged-security structured-extra reflection and `python -m app.worker` logger findings were fixed, and the latest 76-file staged technical/security review again reports 0 Blocker/High/Medium. Independent commit/push, clean-SHA Compose evidence, and exact-SHA GitHub Actions are still pending. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. This does not mark P2-06 repository closeout or Phase 2 complete. +- Recorded two corrected local gate invocations without hiding their first outcomes: the initial integration cleanup command was rejected by the local safety policy before any container started, then the explicit cleanup/run passed; an over-broad Ruff invocation over `scripts/` surfaced 93 existing modernization warnings, while the intended `--select E,F,I` correctness/import gate passed. - On the governance/audit candidate at `665244e095905083b606b8e98e946ed1a02dc0fc`, `make test` passed with `604 passed, 29 skipped` on the backend and `38 passed` on the frontend; a separate real PostgreSQL/Redis integration run passed `29/29`. That frozen implementation also passed `make lint`, offline `make smoke` (`1 passed, 7 deselected`), the frontend production build, isolated SQLite and PostgreSQL migration/check gates, `docker compose config --quiet`, targeted governance/API/Worker suites, enhanced capacity, and 9/9 Compose acceptance. - Pushed the implementation and lock-order fixes through exact SHA `665244e095905083b606b8e98e946ed1a02dc0fc`. Its enhanced capacity evidence (`40deadeb…0588`) records finite policy, exact 4×`202` + 2×`429`, cooperative/cross-Model fairness, fault recovery, 18 Runs/270 Responses/271 reservations/1229 audit events, and empty cleanup; its 9/9 acceptance evidence (`ab311665…ddec`) includes all three deterministic database seams and empty cleanup. GitHub Actions run `33099260233` passed all four required jobs. At that gate, the formal SLO/exporter/retention/backup/Worker-progress closure was still pending. - On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress` for P2-06/P2-07. diff --git a/README.md b/README.md index 526aff4..5101883 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ GitHub：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测工作台。它把模型注册、版本化 Benchmark、后台评测、逐题证据、汇总指标和排行榜放进一条可审计的本地流程，并以“默认离线、严格评分、结果可复现”为首要约束。 -当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选：PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger 与 typed audit 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 +当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选，并实现待仓库级收尾的 P2-06 可观测性/审计保留切片：PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger、typed audit 与 Worker progress 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 ## 当前状态 @@ -17,7 +17,7 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - Phase 2 候选已通过真实 PostgreSQL/Redis 和进程故障验证：除租约、心跳、fencing、幂等 Response 与数据库恢复外，Web/API managed Run 还具有 global/provider/model/run 四层数据库 admission、fixed-minute RPM/TPM、lifetime request/Token/cost budget、有限 backlog、公平 slice、逐 attempt reservation/settlement、typed audit 和历史延迟。 - 可信本地 CLI 已提供 MMLU-Pro 与 GPQA-Diamond 的固定来源转换、真实 OpenAI-compatible 预检、可恢复执行和完整报告导出；这是 Phase 3 的客观题垂直切片，不代表 Phase 2 或 Phase 3 已完成。 - 自动化、CI、Compose 故障验收和容量演练的模型执行都只使用 Mock；根据层级使用临时 SQLite 或隔离 PostgreSQL 16/Redis 7，不访问真实模型服务，也不产生模型费用。 -- Phase 2 仍为 `in_progress`，但 P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。该结论仅限固定 Mock-only 单机控制面，不是生产或真实 Provider SLA；下一任务为 P2-06/P2-07 的 Worker 主循环 progress/liveness、exporter/告警、保留期归档、backup/restore 和剩余恢复矩阵。Phase 3 只交付上述客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 +- Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。P2-06 已在当前工作树实现 `0005`/13 表 importer、Worker DB-time progress、固定低基数 Prometheus exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理；当前 lint、全量 test（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke、真实 PostgreSQL 16/Redis 7 integration（`33 passed, 0 skipped`）、隔离 migration、frontend build、Compose config、Prometheus `v3.5.0` 的八规则 `promtool` 校验，以及 dirty 工作树 9/9 Compose acceptance 与最新 capacity 均已通过。此前 code review 为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。独立 commit/push、clean-SHA Compose 证据和该精确 SHA 的远程 CI 尚未完成，因此 P2-06 仓库级收尾仍为 `in_progress`。P2-07 的数据库/keyring 配对 backup/restore、Redis 重建和剩余故障矩阵仍未完成。上述资格只限固定 Mock-only 单机控制面，不是生产或真实 Provider SLA；Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 最新、可复核的完成状态与测试证据以 [`docs/PROJECT_STATUS.md`](docs/PROJECT_STATUS.md) 和 [`docs/worklogs/`](docs/worklogs/) 为准；Roadmap 中的计划能力不等于已交付能力。 @@ -34,7 +34,7 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - **幂等与恢复**：同一 Run/Question 只有一条计分证据；租约心跳、有限 attempt、退避、取消、过期接管和 dead-letter 都由数据库裁决，Redis 不是状态数据库。 - **数据库权威治理**：Web/API admission 把版本化完整 policy 冻结进 Run；global/provider/model/run 四层并发、RPM/TPM 和累计预算在固定锁序中共同裁决，backlog 满时在提交前稳定拒绝，Token/cost hard limit 缺少显式上界或价格时 fail closed。 - **逐 Provider attempt 账本与公平调度**：每次 HTTP attempt 先 reserve、再持久化 `send_started`、最后 actual/conservative settlement；可证明未发送的 release 保留终态 ledger，另起 generation 并重试当前未发送 ordinal，不重置之前已发送的 HTTP retry。Worker 每个 lease 只新增有界 question quantum，按最久未获服务顺序 cooperative yield，不把让出误计为失败。 -- **可审计观测**：typed、应用 append-only audit 以稳定 event key 去重；`/tasks/history` 在同一读取快照中校验 retained audit 后给出 counters（pending/running cancel 都计 `run_cancel_requested`，dead-letter 单列 `run_dead_lettered`）与 Run queue/execution/end-to-end p50/p95/p99；Run created/finished 和 credential audit 统一取数据库 UTC。逐题保存经过安全归一化的 Provider request/model/fingerprint/finish/attempt 元数据，credential 生命周期写入不含 Key/origin/envelope 的 security audit。 +- **可审计观测与受控保留**：typed、应用 append-only audit 以稳定 event key 去重；`/tasks/history` 在同一读取快照中校验 retained audit 后给出 counters 与 Run latency，`/metrics/prometheus` 用固定 gauge/enum label、硬样本上限和进程内 single-flight 暴露同源快照。Worker generation 只在真实 scan/claim/lease-heartbeat/progress 后按数据库 UTC 合并刷新，dependency probe 仍明确不检查主循环。八条 Prometheus 规则附固定 Runbook；`llmbenchlab-audit-retention` 提供 canonical JSONL archive、离线 verify、reconcile、精确 restore/delete，默认不删除且不把普通 hash 冒充 WORM。Run created/finished、credential audit 和逐题 Provider 元数据继续遵守非秘密边界。 - **可复现记录**：持久化模型参数、Prompt、Benchmark Hash、协议版本、代码 commit（可用时）、raw response、parsed answer、参考答案快照和逐题评分。 - **七个前端页面**：Dashboard、Models、Benchmarks、Evaluation Runs、New Run、Run Detail 和 Leaderboard；评测记录页可找回全部状态的 Run，详情证据按 100 条分页。Run Detail 会明确区分 `managed`、`delayed`、`exhausted` 和 `legacy_unmanaged`，对可公开的稳定 reason 给出中文说明并以 UTC 显示最早重调度时间；未知 reason 不原样反射。 - **Web 只写凭据**：用户可在 Models 表单直接粘贴 API Key；API 不把凭据流中的原值复制到公开 Model/Run-model 字段，数据库只保存由独立 keyring 加密的 AES-GCM 密文。旧 `api_key_env` 模型仍兼容；Provider 返回证据会递归检查对象键/JSON 标量，当前 Key 的精确回显会在进入 Runner/持久化前替换为 `[REDACTED]`。这不是对无关 Benchmark/Question 内容的全局字面扫描。 @@ -149,11 +149,12 @@ curl -sS http://127.0.0.1:8000/api/v1/health curl -sS http://127.0.0.1:8000/api/v1/ready curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics curl -sS 'http://127.0.0.1:8000/api/v1/tasks/history?window_hours=24' +curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus ``` -`/live` 不访问外部依赖，`/health` 仅检查数据库，`/ready` 检查数据库、Alembic head 和 Redis。Redis 不可用时 `/ready` 返回 `503/degraded`，但数据库可用时 API 仍可提交 Run，Worker 也可仅靠数据库对账恢复。`/ready` 对 `asyncio.to_thread` 的等待超时不会取消已进入线程的同步数据库驱动调用，真正资源上界仍由驱动/连接池 timeout 约束。`/tasks/metrics` 是数据库当前 gauges；`/tasks/history` 从 retained typed audit 和 Run 时间字段聚合历史 counters/延迟，单 Run audit 由 `/runs/{id}/audit` 稳定分页。它们都不是 Prometheus exporter、告警发送器或 WORM 证据。 +`/live` 不访问外部依赖，`/health` 仅检查数据库，`/ready` 检查数据库、Alembic head 和 Redis。Redis 不可用时 `/ready` 返回 `503/degraded`，但数据库可用时 API 仍可提交 Run，Worker 也可仅靠数据库对账恢复。`/ready` 对 `asyncio.to_thread` 的等待超时不会取消已进入线程的同步数据库驱动调用，真正资源上界仍由驱动/连接池 timeout 约束。`/tasks/metrics` 是数据库当前 gauges，并包括 Worker expected/registered/live/stalled/shortfall 和最近活动时间；`/tasks/history` 从 retained typed audit 和 Run 时间字段聚合历史 counters/延迟，单 Run audit 由 `/runs/{id}/audit` 稳定分页。`/metrics/prometheus` 固定输出 Prometheus text `0.0.4` gauge：同一 DB-time 快照、有界 15 分钟 audit/1 小时 latency 窗口、固定 enum label 和每 API 进程 single-flight；它不是状态数据库、告警发送器或 WORM 证据。 -API 为每个请求自行生成 `X-Request-ID` 并在响应中返回，不信任或回显客户端提供的同名 header。LLMBenchLab **应用 logger** 使用字段白名单的脱敏 JSON，关联 request/run/question/worker/attempt/lease 事件；这一保证不涵盖所有 Uvicorn 或 access log handler，因此凭据和敏感内容仍绝不得放在 URL、header 或请求路径中。 +API 为每个请求自行生成 `X-Request-ID` 并在响应中返回，不信任或回显客户端提供的同名 header。LLMBenchLab 生产日志调用只允许无格式参数的字面量消息；结构化 extra 除字段白名单外还逐字段执行固定枚举、UUID/Redis stream ID 与有限数值规范化，非法 ID 被省略，未知 method/code 只输出固定 `unsupported`。Redis Run 通知本身也只接受 canonical UUID。外部 logger 的动态消息不进入 JSON，原始 Uvicorn access handler 关闭。凭据和敏感内容仍绝不得放在 URL、header、请求路径或日志字段中。 如需分别观察日志，可在三个终端运行 `make backend`、`make worker` 和 `make frontend`。只启动 API 时，新 Run 会持久化为 `pending`，但不会在 API 进程内执行。本地 SQLite 只支持一个 Worker；Redis URL 可留空，Worker 将使用数据库对账。所有命令可通过 `make help` 查看；更完整的环境变量、迁移和排障说明见 [`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md)。 @@ -331,7 +332,7 @@ make docker-up - Web（Nginx 同源代理 `/api/`）：`http://127.0.0.1:8080` - API 就绪检查：`http://127.0.0.1:8000/api/v1/ready` -API 和 frontend 的 host ports 明确绑定 loopback，PostgreSQL/Redis 不发布 host port。Worker 容器的 healthcheck 是**依赖能力探针**：数据库或 schema 失败时不健康，Redis 失败时报告 degraded 但仍退出 0，因为数据库对账可用。它不检查 Worker 主循环是否活着，不能当作 event-loop liveness 证明。 +API 和 frontend 的 host ports 明确绑定 loopback，PostgreSQL/Redis 不发布 host port。Worker 容器的 healthcheck 是**依赖能力探针**：数据库或 schema 失败时不健康，Redis 失败时报告 degraded 但仍退出 0，因为数据库对账可用。它不检查 Worker 主循环是否活着，不能当作 event-loop liveness 证明；主循环 scan/claim/lease-heartbeat/progress 由 `worker_processes` 的 DB-time generation 事实和 metrics/exporter 聚合另行观测。 停止服务并保留数据： @@ -349,7 +350,7 @@ docker compose config ## SQLite → PostgreSQL 显式导入 -导入器只支持一次性、单向迁移，不是在线复制。必须先停止 SQLite 源的 API/Worker 和新 Run 创建，排空、取消或终结所有 `pending/running` Run，并准备一个已迁移到当前 Alembic head 的**空、离线 PostgreSQL 目标**。导入器以 SQLite read-only URI 读源，检查 integrity/FK/head/active Run，并拒绝 active reservation 以及 ledger 重算后任一 scope/minute 物化计数的高、低漂移；目标使用 advisory lock、`ACCESS EXCLUSIVE` table locks 和一个事务复制 12 张核心/治理表，包括加密凭据、policy/scope/bucket、question execution、attempt ledger、typed audit 和 Provider metadata。 +导入器只支持一次性、单向迁移，不是在线复制。必须先停止 SQLite 源的 API/Worker 和新 Run 创建，排空、取消或终结所有 `pending/running` Run，并准备一个已迁移到当前 Alembic head 的**空、离线 PostgreSQL 目标**。导入器以 SQLite read-only URI 读源，检查 integrity/FK/head/active Run，并拒绝 active reservation、仍 live 的 Worker generation，以及 ledger 重算后任一 scope/minute 物化计数的高、低漂移；目标使用 advisory lock、`ACCESS EXCLUSIVE` table locks 和一个事务复制 13 张核心/治理/运维事实表，包括加密凭据、policy/scope/bucket、question execution、attempt ledger、typed audit、Provider metadata 与 stopped/stale Worker progress。 带凭据的 PostgreSQL DSN 必须通过受控环境变量提供，不得放入命令行： @@ -370,7 +371,22 @@ uv run python -m app.db.import_sqlite \ | `3` | `committed_but_verification_failed` | 目标已提交完整 precommit 快照，但 postcommit 验证或报告失败；停止服务并独立对账 | | `4` | `commit_outcome_unknown` | PostgreSQL 未确认 COMMIT；原子事务意味着目标可能为空，也可能已完整提交 | -退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、12 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。正常使用后的 `20260827_0004` 也不能原地降级丢弃 ledger/audit；guard 会在 DDL 前拒绝，只有隔离空库用于 `0004 → 0003 → 0004` 往返，完整处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 +退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、13 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。正常使用后的 `20260828_0005` 不能在 `worker_processes` 有事实时原地降级；先停止 Worker、保存必要事实并显式清空该表后，才能进入 `0005 → 0004`，而 `0004` 原有 ledger/audit downgrade guard 仍继续生效。只有隔离空库用于完整降级/升级往返，处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 + +## Audit retention 维护 + +`llmbenchlab-audit-retention` 只处理已达到数据库 `expires_at` 的 typed audit；archive 默认不删除。文件是权限不宽于 `0600` 的 canonical JSONL，带完整 rollup、内容 hash 与整文件 SHA-256，但不是签名或 WORM。推荐顺序固定为 archive → 离线 verify → 维护窗口内 delete： + +```bash +cd backend +uv run llmbenchlab-audit-retention archive --output /secure/existing-dir/audit.jsonl +uv run llmbenchlab-audit-retention verify --archive /secure/existing-dir/audit.jsonl +uv run llmbenchlab-audit-retention delete \ + --archive /secure/existing-dir/audit.jsonl \ + --confirm-sha256 +``` + +`verify` 不创建数据库 engine，也不需要有效 DSN；`reconcile` 用于 delete/restore 的只读精确对账，`restore` 只接受同一已确认 digest。退出码 `3`（提交后验证失败）或 `4`（commit outcome unknown）都禁止盲目重跑，先用同一 archive/digest 执行 `reconcile`。archive 含内部 ID，必须按敏感运维文件保护；完整路径与恢复流程见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 ## Roadmap @@ -378,7 +394,7 @@ uv run python -m app.db.import_sqlite \ | --- | --- | --- | | Phase 0 | 项目治理、需求、架构、协议 | 已完成 | | Phase 1 | FastAPI + React + SQLite 的 MVP 垂直链路 | 已完成 | -| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 的 Mock-only 单机 v2 资格、治理/审计、9/9 Compose acceptance 与精确 SHA CI 已通过；Worker progress/liveness、告警/归档、backup/restore 和剩余恢复矩阵仍待收尾 | +| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 的 Mock-only 单机 v2 资格已闭环；P2-06 已实现且 dirty capacity/9/9 acceptance、修复后 staged 技术/安全终审已通过，尚待 commit/clean-SHA Compose/push/精确 SHA CI；P2-07 backup/restore 和剩余恢复矩阵未完成 | | Phase 3 | 合规标准 Benchmark 与隔离代码评测 | MMLU-Pro/GPQA 客观数据切片已交付；IFEval、沙箱及其余验收未完成 | | Phase 4 | LLM Judge、人工校准与 Arena | 计划中 | | Phase 5 | Agent、工具调用与 Live Benchmark | 计划中 | @@ -413,7 +429,7 @@ uv run python -m app.db.import_sqlite \ - `direct`/`official_cot`、任何 group/limit、GPQA shuffle seed、源 revision、转换器版本或 Dataset Hash 不同的结果都不能直接比较；公共题目污染和供应商同名模型滚动更新仍会限制结论。 - 评分仅含三个确定性客观 Evaluator；没有 LLM Judge、人工评审、Arena、Agent 或 Live Benchmark。 - Dataset Hash 用于一致性检查，不是发布者签名，也不能证明数据没有污染。 -- `/tasks/metrics`、retained `/tasks/history` 与单 Run typed audit 已实现，但没有 Prometheus exporter、告警发送器、trace、WORM/数据库管理员防篡改或自动保留期清理；`resume` 的新 canary 仍不会追加独立事件。 +- `/tasks/metrics`、retained `/tasks/history`、单 Run typed audit、固定 Prometheus exporter、八条规则和显式 audit retention CLI 已实现；仓库仍不部署 Prometheus/Alertmanager、告警发送器或 trace，也不提供 WORM/数据库管理员防篡改、自动请求链路清理或对象存储上传。`resume` 的新 canary 仍不会追加独立事件。 - 逐题 Provider request ID、returned model、system fingerprint、finish reason 和 HTTP attempt count 已按字符/长度/秘密规则 fail closed 持久化并可导出；它们是关联证据，不是供应商真实性或账单证明。 - 已保留 dirty-worktree 增强前基线、clean `665244e…` 容量候选和永久 `unqualified` 的 v1 资格历史；clean `b6a35fe…` 的 v2 `qualified` 结果也只证明固定 Mock 配置下的单机本地控制面，不是真实 Provider、生产 SLO/SLA、HA 或灾难恢复证明。仍无 SBOM 和生产部署支持。 diff --git a/backend/alembic/versions/20260828_0005_worker_progress_retention.py b/backend/alembic/versions/20260828_0005_worker_progress_retention.py new file mode 100644 index 0000000..f6cb112 --- /dev/null +++ b/backend/alembic/versions/20260828_0005_worker_progress_retention.py @@ -0,0 +1,94 @@ +"""Add Worker progress facts and bounded audit scan indexes. + +Revision ID: 20260828_0005 +Revises: 20260827_0004 +Create Date: 2026-08-28 08:00:00 UTC +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "20260828_0005" +down_revision: str | None = "20260827_0004" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + table = "worker_processes" + op.create_table( + table, + sa.Column("generation_id", sa.String(length=36), nullable=False), + sa.Column("worker_id", sa.String(length=128), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_scan_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_claim_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_lease_heartbeat_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "length(worker_id) >= 1 AND length(worker_id) <= 128", + name=op.f("ck_worker_processes_worker_id_length"), + ), + sa.CheckConstraint( + "last_seen_at >= started_at", + name=op.f("ck_worker_processes_seen_after_start"), + ), + *( + sa.CheckConstraint( + f"{column} IS NULL OR ({column} >= started_at AND {column} <= last_seen_at)", + name=op.f(f"ck_worker_processes_{column}_within_seen"), + ) + for column in ( + "last_scan_at", + "last_claim_at", + "last_progress_at", + "last_lease_heartbeat_at", + ) + ), + sa.CheckConstraint( + "stopped_at IS NULL OR stopped_at >= last_seen_at", + name=op.f("ck_worker_processes_stopped_after_seen"), + ), + sa.PrimaryKeyConstraint("generation_id", name=op.f("pk_worker_processes")), + sa.UniqueConstraint("worker_id", name=op.f("uq_worker_processes_worker_id")), + ) + op.create_index( + "ix_worker_processes_stopped_seen_generation", + table, + ["stopped_at", "last_seen_at", "generation_id"], + unique=False, + ) + op.create_index( + "ix_audit_events_expires_id", + "audit_events", + ["expires_at", "id"], + unique=False, + ) + op.create_index( + "ix_audit_events_occurred_id", + "audit_events", + ["occurred_at", "id"], + unique=False, + ) + + +def downgrade() -> None: + connection = op.get_bind() + count = connection.execute(sa.text("SELECT COUNT(*) FROM worker_processes")).scalar_one() + if count: + raise RuntimeError( + "Cannot downgrade Worker progress schema while process facts exist; " + "stop Workers, preserve required facts, and explicitly clear the table first" + ) + op.drop_index("ix_audit_events_occurred_id", table_name="audit_events") + op.drop_index("ix_audit_events_expires_id", table_name="audit_events") + op.drop_index( + "ix_worker_processes_stopped_seen_generation", + table_name="worker_processes", + ) + op.drop_table("worker_processes") diff --git a/backend/app/api/v1/health.py b/backend/app/api/v1/health.py index 7077e04..0c17c04 100644 --- a/backend/app/api/v1/health.py +++ b/backend/app/api/v1/health.py @@ -2,16 +2,14 @@ import asyncio import logging -import math from collections.abc import Sequence -from datetime import UTC, datetime, timedelta +from datetime import timedelta from typing import Annotated, Any from fastapi import APIRouter, HTTPException, Query, Request, status from fastapi.responses import JSONResponse -from sqlalchemy import case, func, select, text +from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import Session from app.api.deps import SessionDep, SettingsDep from app.core.config import get_settings @@ -19,20 +17,14 @@ from app.core.time import utc_now from app.db.prepare_migrations import database_heads, expected_database_heads from app.db.session import SessionLocal -from app.governance import ( - AuditIntegrityError, - validate_audit_identity_for_read, - validate_audit_payload_for_read, -) -from app.models import ( - AuditEvent, - AuditRetentionClass, - EvaluationRun, - GovernanceRunStatus, - GovernanceScope, - ProviderCallReservation, - ProviderCallReservationState, - RunStatus, +from app.governance import AuditIntegrityError +from app.observability import ( + METRICS_LATENCY_SAMPLE_LIMIT, + collect_task_current, + collect_task_history, + configure_read_snapshot, + database_clock, + latency_summary, ) from app.schemas.system import ( HealthResponse, @@ -50,182 +42,22 @@ logger = logging.getLogger(__name__) _HISTORY_MAX_WINDOW_HOURS = 90 * 24 -_HISTORY_LATENCY_SAMPLE_LIMIT = 10_000 -_TASK_EVENT_TYPES = ( - "governance_policy_bootstrapped", - "governance_policy_applied", - "run_admitted", - "run_claimed", - "run_cancel_requested", - "run_deferred", - "run_yielded", - "run_terminal", - "run_retry_scheduled", - "run_dead_lettered", - "run_lease_reconciled", - "provider_attempt_reserved", - "provider_attempt_send_started", - "provider_attempt_settled", - "question_evidence_persisted", - "queue_notification", - "governance_integrity_error", -) - - -def _as_utc(value: datetime) -> datetime: - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) - - -def _database_clock(session: Session) -> datetime: - value = session.scalar(select(func.current_timestamp())) - if not isinstance(value, datetime): - raise RuntimeError("Database did not return a timestamp for task metrics") - return _as_utc(value) - - -def _percentile(values: Sequence[float], percentage: float) -> float: - ordered = sorted(float(value) for value in values) - if not ordered: - raise ValueError("percentile requires at least one sample") - if len(ordered) == 1: - return ordered[0] - position = (len(ordered) - 1) * percentage / 100 - lower = int(position) - upper = min(lower + 1, len(ordered) - 1) - fraction = position - lower - return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction +_HISTORY_LATENCY_SAMPLE_LIMIT = METRICS_LATENCY_SAMPLE_LIMIT def _latency_summary(values: Sequence[float]) -> TaskLatencyPercentiles: - truncated = len(values) > _HISTORY_LATENCY_SAMPLE_LIMIT - selected = list(values[:_HISTORY_LATENCY_SAMPLE_LIMIT]) - if not selected: - return TaskLatencyPercentiles( - sample_count=0, - truncated=False, - p50_ms=None, - p95_ms=None, - p99_ms=None, - ) + """Compatibility wrapper for the public history response schema.""" + + snapshot = latency_summary(values, sample_limit=_HISTORY_LATENCY_SAMPLE_LIMIT) return TaskLatencyPercentiles( - sample_count=len(selected), - truncated=truncated, - p50_ms=round(_percentile(selected, 50), 6), - p95_ms=round(_percentile(selected, 95), 6), - p99_ms=round(_percentile(selected, 99), 6), + sample_count=snapshot.sample_count, + truncated=snapshot.truncated, + p50_ms=snapshot.p50_ms, + p95_ms=snapshot.p95_ms, + p99_ms=snapshot.p99_ms, ) -def _run_latency_summary( - session: Session, - *, - observed_at: Any, - started_at: Any, - ended_at: Any, - window_start: datetime, - window_end: datetime, -) -> TaskLatencyPercentiles: - rows = session.execute( - select(started_at, ended_at) - .where( - observed_at >= window_start, - observed_at < window_end, - started_at.is_not(None), - ended_at.is_not(None), - ended_at >= started_at, - ) - .order_by(observed_at, EvaluationRun.id) - .limit(_HISTORY_LATENCY_SAMPLE_LIMIT + 1) - ).all() - values = [ - (_as_utc(end_value) - _as_utc(start_value)).total_seconds() * 1000 - for start_value, end_value in rows - ] - return _latency_summary(values) - - -def _configure_task_history_snapshot(session: Session) -> None: - """Start the strongest portable read snapshot before any history query.""" - - dialect = session.get_bind().dialect.name - connection = session.connection() - if dialect == "sqlite": - # SQLite's legacy driver mode does not begin a transaction for SELECT. - connection.exec_driver_sql("BEGIN") - elif dialect == "postgresql": - connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") - - -def _validate_task_history_event(event: AuditEvent) -> str: - """Validate every retained fact before it contributes to a counter.""" - - try: - event_type = event.event_type - validate_audit_payload_for_read(event_type, event.payload, event.payload_hash) - validate_audit_identity_for_read("id", event.id, maximum=36) - validate_audit_identity_for_read("event_key", event.event_key, maximum=255) - validate_audit_identity_for_read( - "correlation_id", - event.correlation_id, - maximum=128, - ) - validate_audit_identity_for_read("run_id", event.run_id, maximum=36) - validate_audit_identity_for_read("model_id", event.model_id, maximum=36) - validate_audit_identity_for_read("question_id", event.question_id, maximum=36) - validate_audit_identity_for_read("worker_id", event.worker_id, maximum=128) - validate_audit_identity_for_read( - "reservation_id", - event.reservation_id, - maximum=36, - ) - - if not isinstance(event.retention_class, AuditRetentionClass): - raise ValueError("invalid audit retention class") - if not isinstance(event.occurred_at, datetime) or not isinstance( - event.expires_at, - datetime, - ): - raise ValueError("invalid audit retention timestamps") - minimum_retention = timedelta( - days=365 if event.retention_class == AuditRetentionClass.SECURITY else 90 - ) - if _as_utc(event.expires_at) - _as_utc(event.occurred_at) < minimum_retention: - raise ValueError("invalid audit retention interval") - - if event.attempt is not None and ( - isinstance(event.attempt, bool) - or not isinstance(event.attempt, int) - or not 0 <= event.attempt <= 2**31 - 1 - ): - raise ValueError("invalid audit attempt") - if event.provider_attempt is not None and ( - isinstance(event.provider_attempt, bool) - or not isinstance(event.provider_attempt, int) - or not 1 <= event.provider_attempt <= 2**31 - 1 - ): - raise ValueError("invalid audit provider attempt") - if event.lease_token is not None and ( - isinstance(event.lease_token, bool) - or not isinstance(event.lease_token, int) - or not 0 <= event.lease_token <= 2**63 - 1 - ): - raise ValueError("invalid audit lease token") - if event.duration_ms is not None and ( - isinstance(event.duration_ms, bool) - or not isinstance(event.duration_ms, (int, float)) - or not math.isfinite(event.duration_ms) - or event.duration_ms < 0 - ): - raise ValueError("invalid audit duration") - except AuditIntegrityError: - raise - except (OverflowError, TypeError, ValueError): - raise AuditIntegrityError("retained audit event failed integrity validation") from None - return event_type - - def _database_readiness() -> tuple[str, str, list[str]]: database_status = "ok" schema_status = "ok" @@ -347,165 +179,47 @@ async def readiness( response_model=TaskMetricsResponse, summary="查看数据库派生的任务运维指标", ) -def task_metrics(session: SessionDep) -> TaskMetricsResponse: +def task_metrics(session: SessionDep, settings: SettingsDep) -> TaskMetricsResponse: """Expose database-derived gauges; metrics never override task truth.""" - now = _database_clock(session) - active_run_statuses = (RunStatus.PENDING, RunStatus.RUNNING) - row = session.execute( - select( - func.coalesce( - func.sum(case((EvaluationRun.status == RunStatus.PENDING, 1), else_=0)), 0 - ).label("pending"), - func.coalesce( - func.sum( - case( - ( - (EvaluationRun.status == RunStatus.PENDING) - & ( - EvaluationRun.next_attempt_at.is_(None) - | (EvaluationRun.next_attempt_at <= now) - ) - & ( - EvaluationRun.governance_not_before.is_(None) - | (EvaluationRun.governance_not_before <= now) - ), - 1, - ), - else_=0, - ) - ), - 0, - ).label("due_pending"), - func.coalesce( - func.sum(case((EvaluationRun.status == RunStatus.RUNNING, 1), else_=0)), 0 - ).label("running"), - func.coalesce( - func.sum( - case( - ( - (EvaluationRun.status == RunStatus.RUNNING) - & (EvaluationRun.lease_expires_at <= now), - 1, - ), - else_=0, - ) - ), - 0, - ).label("expired_running"), - func.coalesce( - func.sum( - case( - ( - EvaluationRun.cancellation_requested.is_(True) - & EvaluationRun.status.in_((RunStatus.PENDING, RunStatus.RUNNING)), - 1, - ), - else_=0, - ) - ), - 0, - ).label("active_cancellation_requests"), - func.coalesce( - func.sum( - case( - ( - (EvaluationRun.status == RunStatus.PENDING) - & EvaluationRun.next_attempt_at.is_not(None), - 1, - ), - else_=0, - ) - ), - 0, - ).label("retry_scheduled"), - func.coalesce( - func.sum(case((EvaluationRun.dead_lettered_at.is_not(None), 1), else_=0)), 0 - ).label("dead_lettered"), - func.coalesce( - func.sum( - case((EvaluationRun.last_error == "queue_notification_unavailable", 1), else_=0) - ), - 0, - ).label("runs_with_queue_notification_error"), - func.coalesce( - func.sum( - case( - ( - EvaluationRun.status.in_(active_run_statuses) - & ( - EvaluationRun.governance_status - != GovernanceRunStatus.LEGACY_UNMANAGED - ), - 1, - ), - else_=0, - ) - ), - 0, - ).label("managed_backlog"), - func.coalesce( - func.sum( - case( - (EvaluationRun.governance_status == GovernanceRunStatus.DELAYED, 1), - else_=0, - ) - ), - 0, - ).label("governance_delayed"), - func.coalesce( - func.sum( - case( - (EvaluationRun.governance_status == GovernanceRunStatus.EXHAUSTED, 1), - else_=0, - ) - ), - 0, - ).label("governance_exhausted"), - func.coalesce(func.sum(EvaluationRun.attempt_count), 0).label("total_attempts"), - func.coalesce(func.sum(EvaluationRun.failed_attempt_count), 0).label( - "total_failed_attempts" - ), - func.coalesce(func.sum(EvaluationRun.dispatch_count), 0).label("total_dispatches"), - ) - ).one() - active_provider_attempts = ( - session.scalar( - select(func.count(ProviderCallReservation.id)).where( - ProviderCallReservation.state.in_( - ( - ProviderCallReservationState.RESERVED, - ProviderCallReservationState.SEND_STARTED, - ) - ) - ) + with session.begin(): + configure_read_snapshot(session) + now = database_clock(session) + snapshot = collect_task_current( + session, + now=now, + worker_stale_seconds=settings.worker_progress_stale_seconds, + worker_expected_processes=settings.worker_expected_processes, ) - or 0 - ) - overdrawn_governance_scopes = ( - session.scalar( - select(func.count(GovernanceScope.id)).where(GovernanceScope.overdrawn.is_(True)) - ) - or 0 - ) return TaskMetricsResponse( - pending=int(row.pending), - due_pending=int(row.due_pending), - running=int(row.running), - expired_running=int(row.expired_running), - active_cancellation_requests=int(row.active_cancellation_requests), - retry_scheduled=int(row.retry_scheduled), - dead_lettered=int(row.dead_lettered), - runs_with_queue_notification_error=int(row.runs_with_queue_notification_error), - managed_backlog=int(row.managed_backlog), - governance_delayed=int(row.governance_delayed), - governance_exhausted=int(row.governance_exhausted), - active_provider_attempts=int(active_provider_attempts), - overdrawn_governance_scopes=int(overdrawn_governance_scopes), - total_attempts=int(row.total_attempts), - total_failed_attempts=int(row.total_failed_attempts), - total_dispatches=int(row.total_dispatches), - timestamp=now, + pending=snapshot.pending, + due_pending=snapshot.due_pending, + running=snapshot.running, + expired_running=snapshot.expired_running, + active_cancellation_requests=snapshot.active_cancellation_requests, + retry_scheduled=snapshot.retry_scheduled, + dead_lettered=snapshot.dead_lettered, + runs_with_queue_notification_error=snapshot.runs_with_queue_notification_error, + managed_backlog=snapshot.managed_backlog, + governance_delayed=snapshot.governance_delayed, + governance_exhausted=snapshot.governance_exhausted, + active_provider_attempts=snapshot.active_provider_attempts, + overdrawn_governance_scopes=snapshot.overdrawn_governance_scopes, + total_attempts=snapshot.total_attempts, + total_failed_attempts=snapshot.total_failed_attempts, + total_dispatches=snapshot.total_dispatches, + worker_expected_processes=snapshot.worker.expected, + worker_registered_processes=snapshot.worker.registered, + worker_live_processes=snapshot.worker.live, + worker_stalled_processes=snapshot.worker.stalled, + worker_shortfall_processes=snapshot.worker.shortfall, + worker_stale_after_seconds=snapshot.worker.stale_seconds, + worker_last_seen_at=snapshot.worker.last_seen_at, + worker_last_scan_at=snapshot.worker.last_scan_at, + worker_last_claim_at=snapshot.worker.last_claim_at, + worker_last_progress_at=snapshot.worker.last_progress_at, + worker_last_lease_heartbeat_at=snapshot.worker.last_lease_heartbeat_at, + timestamp=snapshot.timestamp, ) @@ -525,52 +239,19 @@ def task_history( try: with session.begin(): - _configure_task_history_snapshot(session) - window_end = _database_clock(session) + configure_read_snapshot(session) + window_end = database_clock(session) window_start = window_end - timedelta(hours=window_hours) - events = list( - session.scalars( - select(AuditEvent) - .where( - AuditEvent.occurred_at >= window_start, - AuditEvent.occurred_at < window_end, - ) - .order_by(AuditEvent.occurred_at, AuditEvent.id) - ) - ) - event_values = {event_type: 0 for event_type in _TASK_EVENT_TYPES} - for event in events: - event_type = _validate_task_history_event(event) - if event_type in event_values: - event_values[event_type] += 1 - event_counts = TaskEventCounts( - total=sum(event_values.values()), - **event_values, - ) - - queue_latency = _run_latency_summary( - session, - observed_at=EvaluationRun.started_at, - started_at=EvaluationRun.created_at, - ended_at=EvaluationRun.started_at, - window_start=window_start, - window_end=window_end, - ) - execution_latency = _run_latency_summary( + snapshot = collect_task_history( session, - observed_at=EvaluationRun.finished_at, - started_at=EvaluationRun.started_at, - ended_at=EvaluationRun.finished_at, window_start=window_start, window_end=window_end, + audit_event_limit=None, + latency_sample_limit=_HISTORY_LATENCY_SAMPLE_LIMIT, ) - end_to_end_latency = _run_latency_summary( - session, - observed_at=EvaluationRun.finished_at, - started_at=EvaluationRun.created_at, - ended_at=EvaluationRun.finished_at, - window_start=window_start, - window_end=window_end, + event_counts = TaskEventCounts( + total=sum(snapshot.event_counts.values()), + **snapshot.event_counts, ) except (AuditIntegrityError, LookupError) as exc: logger.error( @@ -592,9 +273,27 @@ def task_history( window_end=window_end, window_hours=window_hours, event_counts=event_counts, - queue_latency=queue_latency, - execution_latency=execution_latency, - end_to_end_latency=end_to_end_latency, + queue_latency=TaskLatencyPercentiles( + sample_count=snapshot.queue_latency.sample_count, + truncated=snapshot.queue_latency.truncated, + p50_ms=snapshot.queue_latency.p50_ms, + p95_ms=snapshot.queue_latency.p95_ms, + p99_ms=snapshot.queue_latency.p99_ms, + ), + execution_latency=TaskLatencyPercentiles( + sample_count=snapshot.execution_latency.sample_count, + truncated=snapshot.execution_latency.truncated, + p50_ms=snapshot.execution_latency.p50_ms, + p95_ms=snapshot.execution_latency.p95_ms, + p99_ms=snapshot.execution_latency.p99_ms, + ), + end_to_end_latency=TaskLatencyPercentiles( + sample_count=snapshot.end_to_end_latency.sample_count, + truncated=snapshot.end_to_end_latency.truncated, + p50_ms=snapshot.end_to_end_latency.p50_ms, + p95_ms=snapshot.end_to_end_latency.p95_ms, + p99_ms=snapshot.end_to_end_latency.p99_ms, + ), latency_sample_limit=_HISTORY_LATENCY_SAMPLE_LIMIT, timestamp=window_end, ) diff --git a/backend/app/api/v1/observability.py b/backend/app/api/v1/observability.py new file mode 100644 index 0000000..ff1de60 --- /dev/null +++ b/backend/app/api/v1/observability.py @@ -0,0 +1,242 @@ +"""Controlled Prometheus exposition backed by one bounded database snapshot.""" + +from __future__ import annotations + +import asyncio +import logging +from contextlib import suppress +from threading import Lock + +from fastapi import APIRouter, HTTPException, Request, Response, status + +from app.api.deps import SettingsDep +from app.core.config import Settings +from app.db.session import SessionLocal +from app.governance import AuditIntegrityError +from app.observability import MetricsObservationLimitExceeded, OperationalSnapshot +from app.observability.prometheus import ( + PROMETHEUS_CONTENT_TYPE, + PrometheusRenderingError, + render_prometheus, +) +from app.observability.snapshot import collect_operational_snapshot +from app.task_queue import QueueUnavailable + +router = APIRouter(tags=["system"]) +logger = logging.getLogger(__name__) + +_COLLECTION_LOCK = Lock() + + +def _consume_collection_result(task: asyncio.Task[tuple[object, object]]) -> None: + """Retrieve a detached collection result without reflecting its exception.""" + + with suppress(BaseException): + task.result() + + +def _collect_database_snapshot(settings: Settings) -> OperationalSnapshot: + """Open the database session inside the worker thread that owns it.""" + + with SessionLocal() as session: + return collect_operational_snapshot( + session, + worker_stale_seconds=settings.worker_progress_stale_seconds, + worker_expected_processes=settings.worker_expected_processes, + ) + + +async def _observe_queue(request: Request, settings: Settings) -> tuple[bool, bool]: + run_queue = request.app.state.run_queue + if run_queue is None: + return False, False + try: + available = await asyncio.wait_for( + run_queue.ping(), + timeout=settings.redis_operation_timeout_seconds, + ) + except (QueueUnavailable, TimeoutError): + logger.warning( + "Metrics queue observation unavailable", + extra={ + "event": "metrics_queue_unavailable", + "error_code": "queue_unavailable", + "result": "unavailable", + }, + ) + return True, False + except Exception: + logger.warning( + "Metrics queue observation failed", + extra={ + "event": "metrics_queue_check_failed", + "error_code": "queue_check_unavailable", + "result": "unavailable", + }, + ) + return True, False + return True, bool(available) + + +async def _collect_with_gate( + request: Request, + settings: Settings, +) -> tuple[object, object]: + """Own the process gate until both observations really reach a terminal state.""" + + try: + database_task = asyncio.create_task(asyncio.to_thread(_collect_database_snapshot, settings)) + queue_task = asyncio.create_task(_observe_queue(request, settings)) + collection = asyncio.gather( + database_task, + queue_task, + return_exceptions=True, + ) + while True: + try: + return await asyncio.shield(collection) + except asyncio.CancelledError: + # Request cancellation cannot reach this detached owner. If a + # lifecycle cancellation does, do not propagate it into the + # synchronous database thread or release the gate early. + if collection.done(): + return collection.result() + finally: + _COLLECTION_LOCK.release() + + +def _failure(status_code: int, code: str, message: str) -> HTTPException: + return HTTPException(status_code=status_code, detail={"code": code, "message": message}) + + +@router.get( + "/metrics/prometheus", + response_class=Response, + responses={ + 429: {"description": "Another scrape is already collecting"}, + 500: {"description": "Audit integrity or rendering failure"}, + 503: {"description": "Database unavailable or observation cap exceeded"}, + }, + summary="抓取固定低基数 Prometheus 指标", +) +async def prometheus_metrics(request: Request, settings: SettingsDep) -> Response: + """Return one non-cached exposition; Redis remains a non-authoritative observation.""" + + if request.url.query: + raise _failure( + status.HTTP_422_UNPROCESSABLE_CONTENT, + "metrics_query_parameters_not_allowed", + "Metrics scrape does not accept query parameters", + ) + if not _COLLECTION_LOCK.acquire(blocking=False): + raise _failure( + status.HTTP_429_TOO_MANY_REQUESTS, + "metrics_scrape_in_progress", + "A metrics scrape is already in progress", + ) + + try: + collection = asyncio.create_task( + _collect_with_gate(request, settings), + name="prometheus-metrics-collection", + ) + except BaseException: + _COLLECTION_LOCK.release() + raise + collection.add_done_callback(_consume_collection_result) + + database_result, queue_result = await asyncio.shield(collection) + + if isinstance(database_result, (AuditIntegrityError, LookupError)): + logger.error( + "Metrics audit event failed read validation", + extra={ + "event": "metrics_audit_integrity_error", + "error_code": "audit_event_integrity_error", + "result": "rejected", + }, + ) + raise _failure( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "audit_event_integrity_error", + "A retained audit event failed integrity validation", + ) + if isinstance(database_result, MetricsObservationLimitExceeded): + logger.warning( + "Metrics audit observation limit exceeded", + extra={ + "event": "metrics_observation_limit_exceeded", + "error_code": "metrics_observation_limit_exceeded", + "result": "rejected", + }, + ) + raise _failure( + status.HTTP_503_SERVICE_UNAVAILABLE, + "metrics_observation_limit_exceeded", + "Metrics observation limit was exceeded", + ) + if isinstance(database_result, Exception): + logger.error( + "Metrics database collection failed", + extra={ + "event": "metrics_database_unavailable", + "error_code": "metrics_database_unavailable", + "result": "unavailable", + }, + ) + raise _failure( + status.HTTP_503_SERVICE_UNAVAILABLE, + "metrics_database_unavailable", + "Metrics database collection failed", + ) + if not isinstance(database_result, OperationalSnapshot): + raise _failure( + status.HTTP_503_SERVICE_UNAVAILABLE, + "metrics_database_unavailable", + "Metrics database collection failed", + ) + + if isinstance(queue_result, Exception): + queue_configured = request.app.state.run_queue is not None + queue_available = False + logger.warning( + "Metrics queue observation failed", + extra={ + "event": "metrics_queue_check_failed", + "error_code": "queue_check_unavailable", + "result": "unavailable", + }, + ) + else: + queue_configured, queue_available = queue_result + + try: + body = render_prometheus( + database_result, + queue_configured=queue_configured, + queue_available=queue_available, + recovery_alert_seconds=settings.worker_recovery_alert_seconds, + ) + except (KeyError, PrometheusRenderingError, TypeError, ValueError) as exc: + logger.error( + "Metrics rendering failed", + extra={ + "event": "metrics_rendering_failed", + "error_code": "metrics_rendering_error", + "result": "rejected", + }, + ) + raise _failure( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "metrics_rendering_error", + "Metrics rendering failed", + ) from exc + + return Response( + content=body, + status_code=status.HTTP_200_OK, + headers={ + "Content-Type": PROMETHEUS_CONTENT_TYPE, + "Cache-Control": "no-store", + }, + ) diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index 399563f..e37ec02 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -7,10 +7,12 @@ from app.api.v1.health import router as health_router from app.api.v1.leaderboard import router as leaderboard_router from app.api.v1.models import router as models_router +from app.api.v1.observability import router as observability_router from app.api.v1.runs import router as runs_router api_router = APIRouter() api_router.include_router(health_router) +api_router.include_router(observability_router) api_router.include_router(governance_router) api_router.include_router(models_router) api_router.include_router(benchmarks_router) diff --git a/backend/app/api/v1/runs.py b/backend/app/api/v1/runs.py index 8ddbd6c..112bc58 100644 --- a/backend/app/api/v1/runs.py +++ b/backend/app/api/v1/runs.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import math from datetime import timedelta from fastapi import APIRouter, HTTPException, Request, status @@ -19,12 +18,10 @@ GovernanceIntegrityError, GovernanceRepository, record_governance_integrity_event, - validate_audit_identity_for_read, - validate_audit_payload_for_read, ) +from app.governance.audit import validate_audit_event_for_read from app.models import ( AuditEvent, - AuditRetentionClass, Benchmark, EvaluationResponse, EvaluationRun, @@ -96,73 +93,24 @@ def _record_governance_integrity( def _audit_event_read_item(event: AuditEvent) -> dict[str, object]: - minimum_retention = timedelta( - days=365 if event.retention_class == AuditRetentionClass.SECURITY else 90 - ) - if ( - ( - event.attempt is not None - and ( - isinstance(event.attempt, bool) - or not isinstance(event.attempt, int) - or not 0 <= event.attempt <= 2**31 - 1 - ) - ) - or ( - event.provider_attempt is not None - and ( - isinstance(event.provider_attempt, bool) - or not isinstance(event.provider_attempt, int) - or not 1 <= event.provider_attempt <= 2**31 - 1 - ) - ) - or ( - event.lease_token is not None - and ( - isinstance(event.lease_token, bool) - or not isinstance(event.lease_token, int) - or not 0 <= event.lease_token <= 2**63 - 1 - ) - ) - or ( - event.duration_ms is not None - and ( - isinstance(event.duration_ms, bool) - or not isinstance(event.duration_ms, (int, float)) - or not math.isfinite(event.duration_ms) - or event.duration_ms < 0 - ) - ) - or event.expires_at - event.occurred_at < minimum_retention - ): - raise AuditIntegrityError("retained audit event failed integrity validation") + facts = validate_audit_event_for_read(event) return { - "id": validate_audit_identity_for_read("id", event.id, maximum=36), - "event_type": event.event_type, - "payload": validate_audit_payload_for_read( - event.event_type, - event.payload, - event.payload_hash, - ), - "retention_class": event.retention_class, - "occurred_at": event.occurred_at, - "expires_at": event.expires_at, - "correlation_id": validate_audit_identity_for_read( - "correlation_id", event.correlation_id, maximum=128 - ), - "run_id": validate_audit_identity_for_read("run_id", event.run_id, maximum=36), - "model_id": validate_audit_identity_for_read("model_id", event.model_id, maximum=36), - "question_id": validate_audit_identity_for_read( - "question_id", event.question_id, maximum=36 - ), - "worker_id": validate_audit_identity_for_read("worker_id", event.worker_id, maximum=128), - "reservation_id": validate_audit_identity_for_read( - "reservation_id", event.reservation_id, maximum=36 - ), - "attempt": event.attempt, - "provider_attempt": event.provider_attempt, - "lease_token": event.lease_token, - "duration_ms": event.duration_ms, + "id": facts.id, + "event_type": facts.event_type, + "payload": facts.payload, + "retention_class": facts.retention_class, + "occurred_at": facts.occurred_at, + "expires_at": facts.expires_at, + "correlation_id": facts.correlation_id, + "run_id": facts.run_id, + "model_id": facts.model_id, + "question_id": facts.question_id, + "worker_id": facts.worker_id, + "reservation_id": facts.reservation_id, + "attempt": facts.attempt, + "provider_attempt": facts.provider_attempt, + "lease_token": facts.lease_token, + "duration_ms": facts.duration_ms, } @@ -366,18 +314,18 @@ def list_run_audit_events( _get_run_or_404(session, run_id) filters = (AuditEvent.run_id == run_id,) total = session.scalar(select(func.count()).select_from(AuditEvent).where(*filters)) or 0 - events = list( - session.scalars( - select(AuditEvent) - .where(*filters) - .order_by(AuditEvent.occurred_at, AuditEvent.id) - .offset(pagination.offset) - .limit(pagination.limit) - ) - ) try: + events = list( + session.scalars( + select(AuditEvent) + .where(*filters) + .order_by(AuditEvent.occurred_at, AuditEvent.id) + .offset(pagination.offset) + .limit(pagination.limit) + ) + ) items = [_audit_event_read_item(event) for event in events] - except AuditIntegrityError as exc: + except (AuditIntegrityError, LookupError, OverflowError, TypeError, ValueError) as exc: logger.error( "Retained Run audit event failed read validation", extra={ diff --git a/backend/app/cli/audit_retention.py b/backend/app/cli/audit_retention.py new file mode 100644 index 0000000..6498144 --- /dev/null +++ b/backend/app/cli/audit_retention.py @@ -0,0 +1,159 @@ +"""Trusted maintenance CLI for bounded LLMBenchLab audit retention archives.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence + +from app.governance.audit_archive import ( + AuditArchiveError, + confirm_archive_digest, + verify_archive, +) +from app.governance.audit_retention_errors import ( + AuditRetentionCommitOutcomeUnknownError, + AuditRetentionCommittedVerificationError, + AuditRetentionError, +) + + +class AuditRetentionArgumentError(RuntimeError): + """A parser failure whose untrusted argv text must not be reflected.""" + + +class _FixedErrorArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise AuditRetentionArgumentError("audit_retention_argument_invalid") + + +def build_parser() -> argparse.ArgumentParser: + parser = _FixedErrorArgumentParser( + description="Archive and reconcile expired LLMBenchLab typed audit events", + ) + commands = parser.add_subparsers(dest="command", required=True) + + archive = commands.add_parser("archive", help="Write one bounded archive without deleting") + archive.add_argument("--output", required=True) + + verify = commands.add_parser("verify", help="Verify one archive without a database connection") + verify.add_argument("--archive", required=True) + verify.add_argument("--expected-sha256") + + for command in ("reconcile", "restore", "delete"): + mutation = commands.add_parser(command) + mutation.add_argument("--archive", required=True) + mutation.add_argument("--confirm-sha256", required=True) + return parser + + +def _database_factory(): + # Kept lazy so the verify command is a genuinely offline file operation. + from app.db.session import SessionLocal + + return SessionLocal + + +def _run(arguments: argparse.Namespace) -> int: + operation = arguments.command + if operation == "verify": + archive = verify_archive( + arguments.archive, + expected_sha256=arguments.expected_sha256, + ) + print( + f"status=verified operation=verify count={len(archive.events)} " + f"archive_sha256={archive.archive_sha256}" + ) + return 0 + + if operation == "archive": + from app.governance.audit_retention import archive_expired_events + + result = archive_expired_events(_database_factory(), arguments.output) + print( + f"status=archived operation=archive count={result.event_count} " + f"has_more_eligible={str(result.has_more_eligible).lower()} " + f"archive_sha256={result.archive_sha256}" + ) + return 0 + + archive = verify_archive(arguments.archive) + confirm_archive_digest(archive, arguments.confirm_sha256) + factory = _database_factory() + if operation == "reconcile": + from app.governance.audit_retention import reconcile_archive + + result = reconcile_archive(factory, archive) + print( + f"status={result.status} operation=reconcile count={result.event_count} " + f"archive_sha256={archive.archive_sha256}" + ) + return 0 + if operation == "restore": + from app.governance.audit_retention import restore_archive + + result = restore_archive(factory, archive) + elif operation == "delete": + from app.governance.audit_retention import delete_archive_events + + result = delete_archive_events(factory, archive) + else: # argparse constrains this branch. + raise AssertionError("unsupported audit retention command") + print( + f"status={result.status} operation={operation} count={result.event_count} " + f"changed={result.changed_count} archive_sha256={result.archive_sha256}" + ) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + arguments = build_parser().parse_args(argv) + except AuditRetentionArgumentError as exc: + print( + "status=failed operation=parse error_code=audit_retention_argument_invalid " + f"error_type={type(exc).__name__}", + file=sys.stderr, + ) + return 2 + try: + return _run(arguments) + except AuditRetentionCommitOutcomeUnknownError as exc: + count = exc.event_count if exc.event_count is not None else "unknown" + digest = exc.archive_sha256 or "unavailable" + print( + f"status=commit_outcome_unknown operation={arguments.command} " + f"error_code=audit_retention_commit_outcome_unknown " + f"count={count} archive_sha256={digest} error_type={type(exc).__name__}", + file=sys.stderr, + ) + return 4 + except AuditRetentionCommittedVerificationError as exc: + count = exc.event_count if exc.event_count is not None else "unknown" + digest = exc.archive_sha256 or "unavailable" + print( + f"status=committed_but_verification_failed operation={arguments.command} " + f"error_code=audit_retention_postcommit_verification_failed " + f"count={count} archive_sha256={digest} error_type={type(exc).__name__}", + file=sys.stderr, + ) + return 3 + except (AuditArchiveError, AuditRetentionError) as exc: + print( + f"status=failed operation={arguments.command} error_code={exc.code} " + f"error_type={type(exc).__name__}", + file=sys.stderr, + ) + return 2 + except Exception as exc: + print( + f"status=failed operation={arguments.command} " + f"error_code=audit_retention_internal_error error_type={type(exc).__name__}", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index f903940..ce4e324 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -54,6 +54,10 @@ class Settings(BaseSettings): worker_retry_backoff_base_seconds: float = Field(default=1.0, ge=0.0, le=3600.0) worker_retry_backoff_cap_seconds: float = Field(default=30.0, ge=0.0, le=86_400.0) worker_shutdown_grace_seconds: float = Field(default=30.0, ge=0.0, le=3600.0) + worker_progress_flush_seconds: float = Field(default=5.0, ge=1.0, le=300.0) + worker_progress_stale_seconds: float = Field(default=60.0, ge=3.0, le=86_400.0) + worker_expected_processes: int = Field(default=1, ge=0, le=1000) + worker_recovery_alert_seconds: float = Field(default=60.0, gt=0.0, le=86_400.0) mock_generation_delay_seconds: float = Field(default=0.0, ge=0.0, le=5.0) redis_block_milliseconds: int = Field(default=1000, ge=50, le=60_000) cors_origins: Annotated[list[str], NoDecode] = Field( @@ -134,6 +138,17 @@ def validate_worker_timing(self) -> Self: raise ValueError("worker_heartbeat_seconds must be at most half the lease duration") if self.worker_retry_backoff_base_seconds > self.worker_retry_backoff_cap_seconds: raise ValueError("worker retry backoff base must not exceed its cap") + minimum_stale_seconds = 3 * max( + self.worker_progress_flush_seconds, + self.worker_poll_seconds, + self.redis_block_milliseconds / 1000, + self.worker_heartbeat_seconds, + ) + if self.worker_progress_stale_seconds < minimum_stale_seconds: + raise ValueError( + "worker_progress_stale_seconds must be at least three times the longest " + "Worker progress, poll, Redis block, or heartbeat interval" + ) return self diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py index efaf05c..653577a 100644 --- a/backend/app/core/logging.py +++ b/backend/app/core/logging.py @@ -7,10 +7,41 @@ from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from datetime import UTC, datetime from typing import Any from uuid import uuid4 +from app.core.logging_contract import ( + OMIT as _OMIT, +) +from app.core.logging_contract import ( + is_application_logger, + normalize_application_message, + normalize_correlation_id, + normalize_log_level, + normalize_redis_stream_id, + normalize_request_method, + normalize_request_path, + normalize_structured_field, + normalize_timestamp, + normalize_uuid_identifier, +) + +__all__ = ( + "REQUEST_ID_HEADER", + "SanitizedJsonFormatter", + "configure_logging", + "correlation_scope", + "get_correlation_id", + "get_request_id", + "new_correlation_id", + "normalize_correlation_id", + "normalize_redis_stream_id", + "normalize_request_method", + "normalize_request_path", + "normalize_uuid_identifier", + "request_scope", +) + REQUEST_ID_HEADER = "X-Request-ID" _correlation_id: ContextVar[str | None] = ContextVar("correlation_id", default=None) _request_id: ContextVar[str | None] = ContextVar("request_id", default=None) @@ -30,20 +61,14 @@ "error_code", "component", ) +_EXTERNAL_LOGGER_NAME = "external" +_EXTERNAL_MESSAGE = "External component log event" def new_correlation_id() -> str: return str(uuid4()) -def normalize_correlation_id(value: str | None) -> str | None: - if value is None or not 1 <= len(value) <= 128: - return None - if not all(character.isalnum() or character in "-._:" for character in value): - return None - return value - - def get_correlation_id() -> str | None: return _correlation_id.get() @@ -70,33 +95,44 @@ def request_scope(value: str) -> Iterator[None]: _request_id.reset(token) -def _json_value(value: Any) -> str | int | float | bool | None: - if value is None or isinstance(value, (str, int, float, bool)): - return value - return str(value) - - class SanitizedJsonFormatter(logging.Formatter): - """Emit allowlisted context without exception text, request bodies, or credentials.""" + """Emit only closed-domain structured context and fixed application messages.""" def format(self, record: logging.LogRecord) -> str: + application_record = is_application_logger(record.name) payload: dict[str, Any] = { - "timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(), - "level": record.levelname, - "logger": record.name, - "message": record.getMessage(), + "timestamp": normalize_timestamp(record.created), + "level": normalize_log_level(record.levelname), + "logger": record.name if application_record else _EXTERNAL_LOGGER_NAME, + # Registered application sources are statically restricted to this + # closed literal-message set. External dynamic identity, messages, + # formatting arguments, exception text, and extras are suppressed. + "message": ( + normalize_application_message(record) if application_record else _EXTERNAL_MESSAGE + ), } - request_id = get_request_id() or getattr(record, "request_id", None) - if request_id is not None: - payload["request_id"] = _json_value(request_id) - correlation_id = get_correlation_id() or getattr(record, "correlation_id", None) - if correlation_id is not None: - payload["correlation_id"] = _json_value(correlation_id) - for field in _STRUCTURED_FIELDS: - if hasattr(record, field): - payload[field] = _json_value(getattr(record, field)) - if record.exc_info and record.exc_info[0] is not None: - payload["exception_type"] = record.exc_info[0].__name__ + request_id = get_request_id() + if request_id is None and application_record: + request_id = getattr(record, "request_id", None) + normalized_request_id = normalize_uuid_identifier(request_id) + if normalized_request_id is not None: + payload["request_id"] = normalized_request_id + + correlation_id = get_correlation_id() + if correlation_id is None and application_record: + correlation_id = getattr(record, "correlation_id", None) + normalized_correlation_id = normalize_correlation_id(correlation_id) + if normalized_correlation_id is not None: + payload["correlation_id"] = normalized_correlation_id + + if application_record: + for field in _STRUCTURED_FIELDS: + if hasattr(record, field): + normalized = normalize_structured_field(field, getattr(record, field)) + if normalized is not _OMIT: + payload[field] = normalized + if application_record and isinstance(record.exc_info, tuple) and record.exc_info: + payload["exception_type"] = "suppressed" return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) @@ -104,3 +140,24 @@ def configure_logging(level: str) -> None: handler = logging.StreamHandler() handler.setFormatter(SanitizedJsonFormatter()) logging.basicConfig(level=level, handlers=[handler], force=True) + # Uvicorn installs named handlers before importing the application. Its + # access message can contain a raw query string, while dependency loggers + # may contain URLs or driver errors. Route all of them through the sanitized + # root formatter and disable the redundant raw access log; the application + # middleware already emits a route-template request completion event. + for logger_name in ( + "httpcore", + "httpx", + "redis", + "sqlalchemy", + "sqlalchemy.engine", + "uvicorn", + "uvicorn.error", + ): + external_logger = logging.getLogger(logger_name) + external_logger.handlers.clear() + external_logger.propagate = True + access_logger = logging.getLogger("uvicorn.access") + access_logger.handlers.clear() + access_logger.propagate = False + access_logger.disabled = True diff --git a/backend/app/core/logging_contract.py b/backend/app/core/logging_contract.py new file mode 100644 index 0000000..d6e2ff0 --- /dev/null +++ b/backend/app/core/logging_contract.py @@ -0,0 +1,472 @@ +"""Closed, non-reflective value contract for serialized application logs.""" + +from __future__ import annotations + +import logging +import math +import re +from datetime import UTC, datetime +from uuid import UUID + +OMIT = object() + +_APPLICATION_LOGGER_NAMES = frozenset( + { + "app.api.v1.governance", + "app.api.v1.health", + "app.api.v1.observability", + "app.api.v1.runs", + "app.main", + "app.runners.evaluation_runner", + "app.worker", + "app.worker_progress", + "app.workers.service", + } +) +_APPLICATION_MESSAGE = "Application log event" +_UNSUPPORTED = "unsupported" +_UNSUPPORTED_ROUTE = "" +_LEVELS = frozenset({"CRITICAL", "DEBUG", "ERROR", "INFO", "WARNING"}) +_APPLICATION_MESSAGES = frozenset( + { + "API request completed", + "API request failed", + "Discarding invalid Run queue notification", + "Evaluation adapter cleanup failed", + "Evaluation run drained for shutdown; durable lease expiry will recover it", + "Evaluation run failed", + "Evaluation run failure transition resolved", + "Evaluation run finish transition resolved", + "Evaluation run governance transition resolved", + "Evaluation run interrupted; durable lease expiry will recover it", + "Evaluation run question quantum yielded", + "Evaluation run stopped after cancellation", + "Evaluation run stopped after losing its lease", + "Evaluator failed for question", + "Expired Run reconciliation completed", + "Expired Run reconciliation failed governance integrity validation", + "Governance integrity evidence could not be recorded", + "Metrics audit event failed read validation", + "Metrics audit observation limit exceeded", + "Metrics database collection failed", + "Metrics queue observation failed", + "Metrics queue observation unavailable", + "Metrics rendering failed", + "Question evidence persistence resolved", + "Question processing failed", + "Retained Run audit event failed read validation", + "Retained task history audit event failed read validation", + "Run claim was a durable no-op", + "Run lease claimed", + "Run lease heartbeat failed", + "Run lease heartbeat rejected", + "Run lease heartbeat renewed", + "Run persisted before queue notification", + "Run queue ACK observed", + "Run queue connection recovered", + "Run queue failure evidence could not be recorded", + "Run queue notification published", + "Run queue notification unavailable", + "Run queue success evidence could not be recorded", + "Worker database connection recovered", + "Worker database unavailable; task delivery is paused", + "Worker finished Run handling", + "Worker main loop registered and started", + "Worker main service stopped before a healthy lifecycle completed", + "Worker process is starting", + "Worker progress flush failed; retained for retry", + "Worker progress graceful stop failed; generation will become stale", + "Worker progress observer rejected an execution fact", + "Worker received Run queue delivery", + "Worker Run task escaped its isolation boundary", + "Worker shutdown grace expired; active Run was interrupted", + "Worker started Run handling", + "Worker startup database check failed", + "Worker stopped", + "Run queue unavailable; database reconciliation remains active", + } +) +_HTTP_METHODS = frozenset({"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"}) +_ROUTE_TEMPLATES = frozenset( + { + "", + "/benchmarks", + "/benchmarks/import", + "/benchmarks/reload-demo", + "/benchmarks/{benchmark_id}", + "/benchmarks/{benchmark_id}/questions", + "/docs", + "/docs/oauth2-redirect", + "/governance/policy", + "/health", + "/info", + "/leaderboard", + "/live", + "/metrics/prometheus", + "/metrics/summary", + "/models", + "/models/{model_id}", + "/openapi.json", + "/ready", + "/redoc", + "/runs", + "/runs/{run_id}", + "/runs/{run_id}/audit", + "/runs/{run_id}/cancel", + "/runs/{run_id}/responses", + "/tasks/history", + "/tasks/metrics", + } +) +_EVENTS = frozenset( + { + "api_request_completed", + "api_request_failed", + "governance_integrity_audit_failed", + "metrics_audit_integrity_error", + "metrics_database_unavailable", + "metrics_observation_limit_exceeded", + "metrics_queue_check_failed", + "metrics_queue_unavailable", + "metrics_rendering_failed", + "question_evaluator_failed", + "question_evidence_persisted", + "question_processing_failed", + "run_adapter_cleanup_failed", + "run_attempt_failed", + "run_attempt_failure_resolved", + "run_attempt_finished", + "run_audit_integrity_error", + "run_cancelled", + "run_claim_noop", + "run_claimed", + "run_created", + "run_governance_transition_resolved", + "run_heartbeat_failed", + "run_heartbeat_rejected", + "run_heartbeat_renewed", + "run_lease_lost", + "run_question_quantum_yielded", + "run_queue_ack_failed", + "run_queue_ack_observed", + "run_queue_audit_failed", + "run_queue_delivery_received", + "run_queue_invalid_message", + "run_queue_publish_failed", + "run_queue_published", + "run_queue_read_failed", + "run_queue_recovered", + "run_shutdown_lease_expiry", + "task_history_audit_integrity_error", + "worker_database_recovered", + "worker_main_loop_started", + "worker_progress_flush_failed", + "worker_progress_observer_failed", + "worker_progress_stop_failed", + "worker_reap_database_unavailable", + "worker_reap_governance_integrity_error", + "worker_reap_outcome", + "worker_run_finished", + "worker_run_started", + "worker_run_unhandled_error", + "worker_scan_database_unavailable", + "worker_service_failed", + "worker_shutdown_interrupted", + "worker_start_failed", + "worker_starting", + "worker_stopped", + } +) +_RESULTS = frozenset( + { + "ack_safe", + "acknowledged", + "already_absent", + "already_present", + "cancel_requested", + "cancelled", + "claimed", + "completed", + "cooperative_yield", + "dead_lettered", + "drained", + "error_response", + "failed", + "fence_lost", + "fenced", + "governance_deferred", + "governance_exhausted", + "ignored", + "ignored_after_cleanup_attempt", + "inserted", + "internal_server_error", + "interrupted", + "lease_expiry_recovery", + "lease_lost", + "not_acknowledged", + "not_claimable", + "not_found", + "not_recorded", + "observed_unavailable", + "paused", + "pending", + "published", + "recovered_completed", + "rejected", + "renewed", + "requested", + "retained", + "retry_or_dead_letter", + "retry_scheduled", + "running", + "stale", + "stopped", + "terminal", + "unavailable", + } +) +_ERROR_CODES = frozenset( + { + "audit_event_integrity_error", + "authentication_error", + "connect_timeout", + "empty_response", + "evaluator_internal_error", + "incomplete_provider_stream", + "invalid_provider_response", + "invalid_provider_stream", + "invalid_request", + "metrics_database_unavailable", + "metrics_observation_limit_exceeded", + "metrics_rendering_error", + "missing_api_key", + "mock_configuration_error", + "mock_error", + "network_error", + "network_timeout", + "none", + "output_truncated", + "parse_error", + "provider_4xx", + "provider_5xx", + "provider_http_error", + "provider_response_too_large", + "provider_stream_error", + "question_internal_error", + "queue_check_unavailable", + "queue_unavailable", + "rate_limited", + "read_timeout", + "unsupported_provider_response_encoding", + } +) +_EXCEPTION_ERROR_FAMILIES = frozenset( + { + "adapter_cleanup_error", + "api_error", + "evaluator_internal_error", + "question_internal_error", + "runner_error", + "worker_progress_error", + "worker_run_error", + "worker_service_error", + } +) +_GOVERNANCE_ERROR_CODES = frozenset( + { + "governance_active_policy_missing", + "governance_database_clock_reversed", + "governance_input_bound_unknown", + "governance_lease_fence_lost", + "governance_minute_bucket_counter_drift", + "governance_minute_bucket_missing", + "governance_multiple_active_policies", + "governance_policy_hash_mismatch", + "governance_policy_missing", + "governance_pricing_unknown", + "governance_provider_retry_exhausted", + "governance_question_execution_missing", + "governance_question_retry_cursor_mismatch", + "governance_release_after_send", + "governance_reservation_missing", + "governance_run_override_snapshot_mismatch", + "governance_run_policy_missing", + "governance_run_snapshot_mismatch", + "governance_scope_counter_drift", + "governance_scope_missing", + "governance_settle_before_send", + "governance_settlement_unknown", + "governance_unbounded_output", + } + | { + f"governance_{scope}_{reason}" + for scope in ("global", "model", "provider", "run") + for reason in ("concurrency", "overdrawn", "rpm", "tpm") + } + | { + f"governance_{scope}_{reason}_budget_exhausted" + for scope in ("global", "run") + for reason in ("cost", "request", "token") + } +) +_COMPONENTS = frozenset({"api", "database", "governance", "metrics", "redis", "worker"}) +_REDIS_STREAM_ID = re.compile(r"(0|[1-9][0-9]{0,19})-(0|[1-9][0-9]{0,19})") +_WORKER_ID = re.compile( + r"worker:[A-Za-z0-9](?:[A-Za-z0-9.-]{0,39}):[1-9][0-9]{0,9}:" + r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" +) +_REAP_RESULT = re.compile( + r"cancelled=(0|[1-9][0-9]{0,9}),completed=(0|[1-9][0-9]{0,9})," + r"dead_lettered=(0|[1-9][0-9]{0,9}),retry_scheduled=(0|[1-9][0-9]{0,9})" +) +_INT32_MAX = 2**31 - 1 +_INT64_MAX = 2**63 - 1 +_UINT64_MAX = 2**64 - 1 + + +def is_application_logger(name: object) -> bool: + return isinstance(name, str) and name in _APPLICATION_LOGGER_NAMES + + +def normalize_uuid_identifier(value: object) -> str | None: + """Return a canonical UUID or ``None`` without reflecting invalid input.""" + + if not isinstance(value, str) or len(value) != 36: + return None + try: + normalized = str(UUID(value)) + except (ValueError, AttributeError): + return None + return normalized if value.lower() == normalized else None + + +def normalize_correlation_id(value: object) -> str | None: + return normalize_uuid_identifier(value) + + +def normalize_redis_stream_id(value: object) -> str | None: + """Return a canonical Redis stream entry ID or ``None``.""" + + if not isinstance(value, str): + return None + match = _REDIS_STREAM_ID.fullmatch(value) + if match is None: + return None + milliseconds, sequence = (int(part) for part in match.groups()) + if milliseconds > _UINT64_MAX or sequence > _UINT64_MAX: + return None + return f"{milliseconds}-{sequence}" + + +def normalize_request_method(value: object) -> str: + return value if isinstance(value, str) and value in _HTTP_METHODS else _UNSUPPORTED + + +def normalize_request_path(value: object) -> str: + return value if isinstance(value, str) and value in _ROUTE_TEMPLATES else _UNSUPPORTED_ROUTE + + +def normalize_log_level(value: object) -> str: + return value if isinstance(value, str) and value in _LEVELS else "UNKNOWN" + + +def normalize_timestamp(value: object) -> str: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return "1970-01-01T00:00:00+00:00" + try: + if not math.isfinite(value): + return "1970-01-01T00:00:00+00:00" + return datetime.fromtimestamp(value, UTC).isoformat() + except (OSError, OverflowError, TypeError, ValueError): + return "1970-01-01T00:00:00+00:00" + + +def normalize_application_message(record: logging.LogRecord) -> str: + if not (isinstance(record.args, tuple) and not record.args): + return _APPLICATION_MESSAGE + if isinstance(record.msg, str) and record.msg in _APPLICATION_MESSAGES: + return record.msg + return _APPLICATION_MESSAGE + + +def _normalize_nonnegative_int(value: object, *, maximum: int) -> int | object: + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= maximum: + return OMIT + return value + + +def _normalize_duration(value: object) -> int | float | object: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return OMIT + try: + finite = math.isfinite(value) + except (OverflowError, TypeError, ValueError): + return OMIT + if not finite or not 0 <= value <= _INT64_MAX: + return OMIT + return value + + +def _normalize_worker_id(value: object) -> str | object: + if not isinstance(value, str): + return OMIT + match = _WORKER_ID.fullmatch(value) + if match is None or normalize_uuid_identifier(match.group(1)) is None: + return OMIT + # Hostnames and PIDs are process-controlled but may still carry local + # metadata. Preserve only the random generation suffix in serialized logs. + return f"worker:{match.group(1)}" + + +def _normalize_result(value: object) -> str: + if isinstance(value, str) and value in _RESULTS: + return value + if isinstance(value, str): + match = _REAP_RESULT.fullmatch(value) + if match is not None and all(int(part) <= _INT32_MAX for part in match.groups()): + return value + return _UNSUPPORTED + + +def _normalize_error_code(value: object) -> str: + if not isinstance(value, str): + return _UNSUPPORTED + if value in _ERROR_CODES or value in _GOVERNANCE_ERROR_CODES: + return value + family, separator, _detail = value.partition(":") + if separator and family in _EXCEPTION_ERROR_FAMILIES: + # Exception class names are useful locally but cannot be proven free of + # attacker-selected text at the final serialization boundary. + return family + return _UNSUPPORTED + + +def normalize_structured_field(field: str, value: object) -> object: + if field == "event": + return value if isinstance(value, str) and value in _EVENTS else _UNSUPPORTED + if field in {"run_id", "question_id"}: + return normalize_uuid_identifier(value) or OMIT + if field == "worker_id": + return _normalize_worker_id(value) + if field == "attempt": + return _normalize_nonnegative_int(value, maximum=_INT32_MAX) + if field == "lease_token": + return _normalize_nonnegative_int(value, maximum=_INT64_MAX) + if field == "message_id": + return normalize_redis_stream_id(value) or OMIT + if field == "request_method": + return normalize_request_method(value) + if field == "request_path": + return normalize_request_path(value) + if field == "status_code": + normalized = _normalize_nonnegative_int(value, maximum=599) + return normalized if normalized is not OMIT and normalized >= 100 else OMIT + if field == "duration_ms": + return _normalize_duration(value) + if field == "result": + return _normalize_result(value) + if field == "error_code": + return _normalize_error_code(value) + if field == "component": + return value if isinstance(value, str) and value in _COMPONENTS else _UNSUPPORTED + return OMIT diff --git a/backend/app/db/__init__.py b/backend/app/db/__init__.py index 1747c26..131f835 100644 --- a/backend/app/db/__init__.py +++ b/backend/app/db/__init__.py @@ -1,6 +1,34 @@ -"""Database engine, session, and initialization helpers.""" +"""Database metadata plus lazy runtime session exports. + +Importing a schema module must not construct the process-wide engine. In +particular, the offline audit-archive verifier imports ORM metadata only to +validate the frozen row contract; it must remain independent of database +settings and filesystem paths. Existing ``from app.db import ...`` callers +retain the same public names through module-level lazy attributes. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any from app.db.base import Base -from app.db.session import SessionLocal, engine, get_db + +if TYPE_CHECKING: + from app.db.session import SessionLocal, engine, get_db __all__ = ["Base", "SessionLocal", "engine", "get_db"] + +_SESSION_EXPORTS = frozenset({"SessionLocal", "engine", "get_db"}) + + +def __getattr__(name: str) -> Any: + if name not in _SESSION_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module("app.db.session"), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/backend/app/db/import_sqlite.py b/backend/app/db/import_sqlite.py index 6ba75c2..bd45d14 100644 --- a/backend/app/db/import_sqlite.py +++ b/backend/app/db/import_sqlite.py @@ -7,15 +7,17 @@ import json import math import os +import re import sqlite3 import sys from collections.abc import Iterable, Mapping, Sequence from contextlib import suppress from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from decimal import Decimal from enum import Enum from pathlib import Path +from types import SimpleNamespace from typing import Any, TextIO import sqlalchemy as sa @@ -25,7 +27,9 @@ from sqlalchemy.schema import Table from app import models as _models # noqa: F401 -- registers all mapped tables +from app.core.config import get_settings from app.db.base import Base +from app.db.clock import as_utc from app.db.prepare_migrations import ( SchemaPreparationError, database_heads, @@ -33,6 +37,13 @@ validate_sqlite_schema_fingerprint, ) from app.db.session import create_database_engine +from app.governance.audit import AuditIntegrityError, validate_audit_event_values_for_read +from app.governance.repository import ( + GovernanceIntegrityError, + _policy_snapshot, + _run_override_snapshot, + _validate_policy_integrity, +) CORE_TABLE_NAMES = ( "governance_policies", @@ -47,6 +58,7 @@ "question_executions", "provider_call_reservations", "audit_events", + "worker_processes", ) DEFAULT_TARGET_ENV = "LLMBENCHLAB_DATABASE_URL" @@ -146,6 +158,7 @@ def values(self) -> tuple[int, ...]: ("model_scope_id", "model"), ("run_scope_id", "run"), ) +_OPAQUE_GOVERNANCE_DIGEST = re.compile(r"[a-f0-9]{64}") def _canonical_value(value: Any) -> Any: @@ -460,7 +473,106 @@ def _validate_governance_materializations(connection: Connection) -> None: ) -def preflight_sqlite_source(connection: Connection) -> None: +def _validate_governance_snapshots(connection: Connection) -> None: + """Reject policy hashes and managed Run snapshots that cannot be executed safely.""" + + policies = Base.metadata.tables["governance_policies"] + runs = Base.metadata.tables["evaluation_runs"] + try: + policy_rows = tuple( + dict(row) + for row in connection.execute(sa.select(policies).order_by(policies.c.id)).mappings() + ) + policies_by_id: dict[str, SimpleNamespace] = {} + for row in policy_rows: + policy = SimpleNamespace(**row) + _validate_policy_integrity(policy) + policies_by_id[str(row["id"])] = policy + + for raw_run in connection.execute(sa.select(runs).order_by(runs.c.id)).mappings(): + run = dict(raw_run) + if _enum_text(run["governance_status"]) == "legacy_unmanaged": + continue + policy_id = run["governance_policy_id"] + policy = policies_by_id.get(str(policy_id)) if policy_id is not None else None + snapshot = run["model_parameters_snapshot"] + governance = snapshot.get("governance") if isinstance(snapshot, Mapping) else None + if policy is None or not isinstance(governance, Mapping): + raise GovernanceIntegrityError("governance_run_snapshot_mismatch") + + expected_policy = _policy_snapshot(policy) + expected_keys = set(expected_policy) | { + "provider_scope_key", + "local_admission_only", + "run_overrides", + } + provider_scope = governance.get("provider_scope_key") + run_record = SimpleNamespace(**run) + if ( + set(governance) != expected_keys + or any(governance.get(name) != value for name, value in expected_policy.items()) + or not isinstance(provider_scope, str) + or _OPAQUE_GOVERNANCE_DIGEST.fullmatch(provider_scope) is None + or governance.get("local_admission_only") is not True + or governance.get("run_overrides") != _run_override_snapshot(run_record) + ): + raise GovernanceIntegrityError("governance_run_snapshot_mismatch") + except GovernanceIntegrityError: + raise SQLiteImportError( + "Source SQLite governance policy or managed Run snapshot failed integrity " + "validation; restore the original immutable policy and Run snapshot facts " + "before importing" + ) from None + except Exception: + raise SQLiteImportError( + "Source SQLite governance policy or managed Run snapshot could not be validated" + ) from None + + +def _validate_retained_audit_events(connection: Connection) -> None: + """Apply the live retained-row contract before copying any audit evidence.""" + + audit_events = Base.metadata.tables["audit_events"] + try: + for event in connection.execute( + sa.select(audit_events).order_by(audit_events.c.occurred_at, audit_events.c.id) + ).mappings(): + validate_audit_event_values_for_read( + id=event["id"], + event_key=event["event_key"], + event_type=event["event_type"], + payload_hash=event["payload_hash"], + payload=event["payload"], + retention_class=event["retention_class"], + occurred_at=event["occurred_at"], + expires_at=event["expires_at"], + correlation_id=event["correlation_id"], + run_id=event["run_id"], + model_id=event["model_id"], + question_id=event["question_id"], + worker_id=event["worker_id"], + reservation_id=event["reservation_id"], + attempt=event["attempt"], + provider_attempt=event["provider_attempt"], + lease_token=event["lease_token"], + duration_ms=event["duration_ms"], + ) + except (AuditIntegrityError, LookupError): + raise SQLiteImportError( + "Source SQLite retained audit event failed integrity validation; restore the " + "original typed audit facts before importing" + ) from None + except Exception: + raise SQLiteImportError( + "Source SQLite retained audit events could not be validated" + ) from None + + +def preflight_sqlite_source( + connection: Connection, + *, + worker_stale_seconds: float | None = None, +) -> None: """Validate a read-only, stopped SQLite source before reading application data.""" if connection.dialect.name != "sqlite": @@ -527,7 +639,35 @@ def preflight_sqlite_source(connection: Connection) -> None: "reserved or send_started ledger row before importing" ) + stale_seconds = ( + get_settings().worker_progress_stale_seconds + if worker_stale_seconds is None + else worker_stale_seconds + ) + if stale_seconds <= 0: + raise SQLiteImportError("Worker stale threshold must be positive") + source_now_value = connection.scalar(sa.select(sa.func.current_timestamp())) + if not isinstance(source_now_value, datetime): + raise SQLiteImportError("Source SQLite database clock is unavailable") + source_now = as_utc(source_now_value) + live_worker_processes = connection.scalar( + sa.select(sa.func.count()) + .select_from(Base.metadata.tables["worker_processes"]) + .where( + Base.metadata.tables["worker_processes"].c.stopped_at.is_(None), + Base.metadata.tables["worker_processes"].c.last_seen_at + >= source_now - timedelta(seconds=stale_seconds), + ) + ) + if live_worker_processes: + raise SQLiteImportError( + "Source SQLite contains a live Worker generation; stop every Worker and wait " + "for graceful stop or the configured stale threshold before importing" + ) + + _validate_governance_snapshots(connection) _validate_governance_materializations(connection) + _validate_retained_audit_events(connection) def _require_empty_target(connection: Connection) -> None: diff --git a/backend/app/db/prepare_migrations.py b/backend/app/db/prepare_migrations.py index a944d5c..cd9a6d2 100644 --- a/backend/app/db/prepare_migrations.py +++ b/backend/app/db/prepare_migrations.py @@ -30,6 +30,22 @@ PHASE_1_REVISION = "20260824_0001" RELIABILITY_REVISION = "20260825_0002" CREDENTIAL_REVISION = "20260827_0003" +GOVERNANCE_REVISION = "20260827_0004" + +_WORKER_PROGRESS_TABLES = {"worker_processes"} +_WORKER_PROGRESS_INDEXES = { + "add_index:audit_events.ix_audit_events_expires_id", + "add_index:audit_events.ix_audit_events_occurred_id", + "add_index:worker_processes.ix_worker_processes_stopped_seen_generation", +} +_WORKER_PROGRESS_DIFFERENCE_SET = { + f"add_table:{table_name}" for table_name in _WORKER_PROGRESS_TABLES +} | _WORKER_PROGRESS_INDEXES +_WORKER_PROGRESS_DIFFERENCES = tuple(sorted(_WORKER_PROGRESS_DIFFERENCE_SET)) +_WORKER_PROGRESS_EXISTING_INDEX_NAMES = { + "ix_audit_events_expires_id", + "ix_audit_events_occurred_id", +} _WEB_CREDENTIAL_DIFFERENCES = { "add_table:model_credentials", @@ -111,6 +127,7 @@ "add_fk:evaluation_runs.fk_evaluation_runs_governance_policy_id_governance_policies", "remove_constraint:evaluation_runs.ck_evaluation_runs_attempt_within_limit", } + | _WORKER_PROGRESS_DIFFERENCE_SET ) _GOVERNANCE_DIFFERENCES = tuple(sorted(_GOVERNANCE_DIFFERENCE_SET)) _GOVERNANCE_CHECK_NAMES_BY_TABLE = { @@ -412,6 +429,15 @@ def _validate_sqlite_ddl_modifiers( def _schema_tables_for_revision(schema_revision: str) -> dict[str, sa.Table]: tables = dict(Base.metadata.tables) + if schema_revision in { + LEGACY_REVISION, + PHASE_1_REVISION, + RELIABILITY_REVISION, + CREDENTIAL_REVISION, + GOVERNANCE_REVISION, + }: + for table_name in _WORKER_PROGRESS_TABLES: + tables.pop(table_name, None) if schema_revision in {LEGACY_REVISION, PHASE_1_REVISION, RELIABILITY_REVISION}: tables.pop("model_credentials", None) if schema_revision in { @@ -460,6 +486,13 @@ def _validate_relational_structure(connection: Connection, *, schema_revision: s RELIABILITY_REVISION, CREDENTIAL_REVISION, } + pre_worker_progress = schema_revision in { + LEGACY_REVISION, + PHASE_1_REVISION, + RELIABILITY_REVISION, + CREDENTIAL_REVISION, + GOVERNANCE_REVISION, + } for table_name, table in _schema_tables_for_revision(schema_revision).items(): table_sql = connection.exec_driver_sql( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", @@ -564,6 +597,11 @@ def _validate_relational_structure(connection: Connection, *, schema_revision: s and table_name == "evaluation_runs" and str(index.name) in _GOVERNANCE_EXISTING_INDEX_NAMES ) + and not ( + pre_worker_progress + and table_name == "audit_events" + and str(index.name) in _WORKER_PROGRESS_EXISTING_INDEX_NAMES + ) } actual_indexes = { ( @@ -786,6 +824,7 @@ def prepare_database( PHASE_1_REVISION, RELIABILITY_REVISION, CREDENTIAL_REVISION, + GOVERNANCE_REVISION, ) historical_heads = {(revision,) for revision in historical_revisions} @@ -842,7 +881,11 @@ def prepare_database( elif source_revision == RELIABILITY_REVISION: expected_differences = _RELIABILITY_DIFFERENCES else: - expected_differences = _CREDENTIAL_DIFFERENCES + expected_differences = ( + _CREDENTIAL_DIFFERENCES + if source_revision == CREDENTIAL_REVISION + else _WORKER_PROGRESS_DIFFERENCES + ) differences = _schema_differences(connection) if differences != expected_differences: rendered_differences = ", ".join(sorted(differences)) or "unknown" @@ -857,6 +900,7 @@ def prepare_database( PHASE_1_REVISION: "versioned_phase1", RELIABILITY_REVISION: "versioned_reliability", CREDENTIAL_REVISION: "versioned_credentials", + GOVERNANCE_REVISION: "versioned_governance", }[source_revision] else: if not sqlite_locked: @@ -881,6 +925,10 @@ def prepare_database( target_revision = CREDENTIAL_REVISION action = "stamped_credentials" source_revision = CREDENTIAL_REVISION + elif differences == _WORKER_PROGRESS_DIFFERENCES: + target_revision = GOVERNANCE_REVISION + action = "stamped_governance" + source_revision = GOVERNANCE_REVISION elif differences == _LEGACY_DIFFERENCES: target_revision = LEGACY_REVISION action = "stamped_legacy" @@ -949,6 +997,11 @@ def main() -> int: "Adopted a verified current SQLite schema at " f"{result.stamped_revision}; backup: {result.backup_path}" ) + elif result.action == "stamped_governance": + print( + "Adopted a verified governance/audit SQLite schema at " + f"{result.stamped_revision}; backup: {result.backup_path}" + ) elif result.action == "versioned_legacy": print( "Verified a versioned legacy SQLite schema before upgrade; " @@ -969,6 +1022,11 @@ def main() -> int: "Verified a versioned Web-credential SQLite schema before upgrade; " f"backup: {result.backup_path}" ) + elif result.action == "versioned_governance": + print( + "Verified a versioned governance/audit SQLite schema before upgrade; " + f"backup: {result.backup_path}" + ) return 0 diff --git a/backend/app/governance/__init__.py b/backend/app/governance/__init__.py index 6debd56..ae16914 100644 --- a/backend/app/governance/__init__.py +++ b/backend/app/governance/__init__.py @@ -1,25 +1,40 @@ -"""Database-authoritative Provider governance and typed audit services.""" - -from .audit import ( - AuditIntegrityError, - append_audit_event, - validate_audit_identity_for_read, - validate_audit_payload_for_read, -) -from .integrity import record_governance_integrity_event -from .repository import ( - DatabaseProviderAttemptController, - GovernanceBacklogFull, - GovernanceDeferred, - GovernanceExhausted, - GovernanceFenceLost, - GovernanceIntegrityError, - GovernanceRepository, - GovernanceSettlementUnknown, - provider_scope_key, -) +"""Lazy public exports for Provider governance and typed audit services. + +The package is imported before any ``app.governance.*`` submodule. Keeping +this boundary lazy lets the archive verifier load its pure validation code +without importing the database-backed repository or constructing an engine. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .audit import ( + AuditEventReadFacts, + AuditIntegrityError, + append_audit_event, + validate_audit_event_for_read, + validate_audit_event_values_for_read, + validate_audit_identity_for_read, + validate_audit_payload_for_read, + ) + from .integrity import record_governance_integrity_event + from .repository import ( + DatabaseProviderAttemptController, + GovernanceBacklogFull, + GovernanceDeferred, + GovernanceExhausted, + GovernanceFenceLost, + GovernanceIntegrityError, + GovernanceRepository, + GovernanceSettlementUnknown, + provider_scope_key, + ) __all__ = [ + "AuditEventReadFacts", "AuditIntegrityError", "DatabaseProviderAttemptController", "GovernanceBacklogFull", @@ -32,6 +47,41 @@ "append_audit_event", "provider_scope_key", "record_governance_integrity_event", + "validate_audit_event_for_read", + "validate_audit_event_values_for_read", "validate_audit_identity_for_read", "validate_audit_payload_for_read", ] + +_EXPORT_MODULES = { + "AuditEventReadFacts": ".audit", + "AuditIntegrityError": ".audit", + "append_audit_event": ".audit", + "validate_audit_event_for_read": ".audit", + "validate_audit_event_values_for_read": ".audit", + "validate_audit_identity_for_read": ".audit", + "validate_audit_payload_for_read": ".audit", + "record_governance_integrity_event": ".integrity", + "DatabaseProviderAttemptController": ".repository", + "GovernanceBacklogFull": ".repository", + "GovernanceDeferred": ".repository", + "GovernanceExhausted": ".repository", + "GovernanceFenceLost": ".repository", + "GovernanceIntegrityError": ".repository", + "GovernanceRepository": ".repository", + "GovernanceSettlementUnknown": ".repository", + "provider_scope_key": ".repository", +} + + +def __getattr__(name: str) -> Any: + module_name = _EXPORT_MODULES.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(module_name, __name__), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/backend/app/governance/audit.py b/backend/app/governance/audit.py index 91ac927..6c6285d 100644 --- a/backend/app/governance/audit.py +++ b/backend/app/governance/audit.py @@ -14,8 +14,11 @@ import json import math import re -from datetime import datetime, timedelta +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta from decimal import Decimal +from types import MappingProxyType from typing import Any from uuid import uuid4 @@ -33,63 +36,124 @@ class AuditIntegrityError(RuntimeError): _SAFE_TOKEN = re.compile(r"[A-Za-z0-9_.:@/-]{1,128}") +_SAFE_IDENTITY_TOKEN = re.compile(r"[A-Za-z0-9_.:@/-]{1,255}") _SAFE_HASH = re.compile(r"[a-f0-9]{64}") _SAFE_DECIMAL = re.compile(r"0|[1-9][0-9]{0,19}(?:\.[0-9]{1,8})?|0\.[0-9]{1,8}") _INT32_MAX = 2**31 - 1 _INT64_MAX = 2**63 - 1 -# Exact event-type payload boundaries. Fields are intentionally numeric or -# short enum-like tokens. An empty set means the transition columns themselves -# are sufficient evidence and no JSON payload is accepted. -_EVENT_PAYLOAD_FIELDS: dict[str, frozenset[str]] = { - "governance_policy_bootstrapped": frozenset({"policy_version", "policy_hash"}), - "governance_policy_applied": frozenset({"policy_version", "policy_hash"}), - "run_admitted": frozenset({"policy_version", "backlog_count", "question_quantum"}), - "run_claimed": frozenset({"dispatch_count"}), - "run_cancel_requested": frozenset(), - "run_deferred": frozenset({"reason", "not_before"}), - "run_yielded": frozenset({"responses_added"}), - "run_terminal": frozenset({"status", "reason"}), - "run_retry_scheduled": frozenset({"failed_attempt_count", "reason"}), - "run_dead_lettered": frozenset({"failed_attempt_count", "reason"}), - "run_lease_reconciled": frozenset({"released_reservations", "conservative_settlements"}), - "provider_attempt_reserved": frozenset( - { - "reserved_input_tokens", - "reserved_output_tokens", - "reserved_cost_usd", - } - ), - "provider_attempt_send_started": frozenset(), - "provider_attempt_settled": frozenset( - { - "disposition", - "outcome", - "input_tokens", - "output_tokens", - "cost_usd", - "reconciled", + +@dataclass(frozen=True) +class AuditEventReadFacts: + """Validated, exact storage facts for one retained audit event. + + Unlike the business replay comparison in :func:`_event_matches`, this + projection includes the primary key and both timestamps. Retention + archive reconciliation can therefore compare every persisted fact without + weakening append-event idempotency semantics. + """ + + id: str + event_key: str + event_type: str + payload_hash: str + payload: dict[str, Any] + retention_class: AuditRetentionClass + occurred_at: datetime + expires_at: datetime + correlation_id: str | None + run_id: str | None + model_id: str | None + question_id: str | None + worker_id: str | None + reservation_id: str | None + attempt: int | None + provider_attempt: int | None + lease_token: int | None + duration_ms: float | None + + def as_insert_values(self) -> dict[str, Any]: + """Return a fresh mapping suitable for an exact ORM insert.""" + + return { + "id": self.id, + "event_key": self.event_key, + "event_type": self.event_type, + "payload_hash": self.payload_hash, + "payload": dict(self.payload), + "retention_class": self.retention_class, + "occurred_at": self.occurred_at, + "expires_at": self.expires_at, + "correlation_id": self.correlation_id, + "run_id": self.run_id, + "model_id": self.model_id, + "question_id": self.question_id, + "worker_id": self.worker_id, + "reservation_id": self.reservation_id, + "attempt": self.attempt, + "provider_attempt": self.provider_attempt, + "lease_token": self.lease_token, + "duration_ms": self.duration_ms, } - ), - "question_evidence_persisted": frozenset({"error_code"}), - "queue_notification": frozenset({"result"}), - "credential_changed": frozenset({"action", "credential_source", "key_id"}), - "credential_rejected": frozenset({"reason", "credential_source", "key_id"}), - "credential_decrypt_failed": frozenset({"reason", "key_id"}), - "governance_integrity_error": frozenset({"reason"}), -} - -_NULLABLE_PAYLOAD_FIELDS: dict[str, frozenset[str]] = { - "provider_attempt_reserved": frozenset( - {"reserved_input_tokens", "reserved_output_tokens", "reserved_cost_usd"} - ), - "provider_attempt_settled": frozenset({"input_tokens", "output_tokens", "cost_usd"}), - "credential_changed": frozenset({"key_id"}), - "credential_rejected": frozenset({"key_id"}), - "credential_decrypt_failed": frozenset({"key_id"}), -} - -_INTEGER_FIELDS = frozenset( + + +# Frozen archive-v1 payload boundaries. The live contract starts as a copy of +# these values so future event evolution cannot silently reinterpret a +# long-lived archive. An empty set means the transition columns themselves are +# sufficient evidence and no JSON payload is accepted. +_ARCHIVE_V1_EVENT_PAYLOAD_FIELDS: Mapping[str, frozenset[str]] = MappingProxyType( + { + "governance_policy_bootstrapped": frozenset({"policy_version", "policy_hash"}), + "governance_policy_applied": frozenset({"policy_version", "policy_hash"}), + "run_admitted": frozenset({"policy_version", "backlog_count", "question_quantum"}), + "run_claimed": frozenset({"dispatch_count"}), + "run_cancel_requested": frozenset(), + "run_deferred": frozenset({"reason", "not_before"}), + "run_yielded": frozenset({"responses_added"}), + "run_terminal": frozenset({"status", "reason"}), + "run_retry_scheduled": frozenset({"failed_attempt_count", "reason"}), + "run_dead_lettered": frozenset({"failed_attempt_count", "reason"}), + "run_lease_reconciled": frozenset({"released_reservations", "conservative_settlements"}), + "provider_attempt_reserved": frozenset( + { + "reserved_input_tokens", + "reserved_output_tokens", + "reserved_cost_usd", + } + ), + "provider_attempt_send_started": frozenset(), + "provider_attempt_settled": frozenset( + { + "disposition", + "outcome", + "input_tokens", + "output_tokens", + "cost_usd", + "reconciled", + } + ), + "question_evidence_persisted": frozenset({"error_code"}), + "queue_notification": frozenset({"result"}), + "credential_changed": frozenset({"action", "credential_source", "key_id"}), + "credential_rejected": frozenset({"reason", "credential_source", "key_id"}), + "credential_decrypt_failed": frozenset({"reason", "key_id"}), + "governance_integrity_error": frozenset({"reason"}), + } +) + +_ARCHIVE_V1_NULLABLE_PAYLOAD_FIELDS: Mapping[str, frozenset[str]] = MappingProxyType( + { + "provider_attempt_reserved": frozenset( + {"reserved_input_tokens", "reserved_output_tokens", "reserved_cost_usd"} + ), + "provider_attempt_settled": frozenset({"input_tokens", "output_tokens", "cost_usd"}), + "credential_changed": frozenset({"key_id"}), + "credential_rejected": frozenset({"key_id"}), + "credential_decrypt_failed": frozenset({"key_id"}), + } +) + +_ARCHIVE_V1_INTEGER_FIELDS = frozenset( { "policy_version", "backlog_count", @@ -105,52 +169,105 @@ class AuditIntegrityError(RuntimeError): "output_tokens", } ) -_DECIMAL_FIELDS = frozenset({"reserved_cost_usd", "cost_usd"}) -_BOOLEAN_FIELDS = frozenset({"reconciled"}) -_HASH_FIELDS = frozenset({"policy_hash"}) -_TIME_FIELDS = frozenset({"not_before"}) -_ENUM_VALUES: dict[str, frozenset[str]] = { - "disposition": frozenset({"released_pre_send", "settled_actual", "settled_conservative"}), - "outcome": frozenset( - { - "succeeded", - "usage_incomplete", - "transport_error", - "http_error", - "provider_response_error", - "cancelled", - "mark_send_failed", - "unexpected_error", - "released_pre_send", - "lease_reconciled_pre_send", - "lease_reconciled_unknown", - } - ), - "reason": frozenset( - { - "none", - "governance_deferred", - "governance_exhausted", - "governance_integrity_error", - "lease_expired", - "origin_rejected", - "active_run_conflict", - "decrypt_failed", - "worker_error", - } - ), - "status": frozenset({"pending", "running", "completed", "failed", "cancelled"}), - "result": frozenset({"published", "unavailable", "acknowledged", "already_absent"}), - "action": frozenset({"created", "replaced", "source_switched", "removed"}), - "credential_source": frozenset({"none", "environment", "stored"}), - "error_code": frozenset( - {"none", "adapter_error", "parse_error", "evaluator_error", "internal_error"} - ), -} - - -def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[str, Any]: - allowed = _EVENT_PAYLOAD_FIELDS.get(event_type) +_ARCHIVE_V1_DECIMAL_FIELDS = frozenset({"reserved_cost_usd", "cost_usd"}) +_ARCHIVE_V1_BOOLEAN_FIELDS = frozenset({"reconciled"}) +_ARCHIVE_V1_HASH_FIELDS = frozenset({"policy_hash"}) +_ARCHIVE_V1_TIME_FIELDS = frozenset({"not_before"}) +_ARCHIVE_V1_ENUM_VALUES: Mapping[str, frozenset[str]] = MappingProxyType( + { + "disposition": frozenset({"released_pre_send", "settled_actual", "settled_conservative"}), + "outcome": frozenset( + { + "succeeded", + "usage_incomplete", + "transport_error", + "http_error", + "provider_response_error", + "cancelled", + "mark_send_failed", + "unexpected_error", + "released_pre_send", + "lease_reconciled_pre_send", + "lease_reconciled_unknown", + } + ), + "reason": frozenset( + { + "none", + "governance_deferred", + "governance_exhausted", + "governance_integrity_error", + "lease_expired", + "origin_rejected", + "active_run_conflict", + "decrypt_failed", + "worker_error", + } + ), + "status": frozenset({"pending", "running", "completed", "failed", "cancelled"}), + "result": frozenset({"published", "unavailable", "acknowledged", "already_absent"}), + "action": frozenset({"created", "replaced", "source_switched", "removed"}), + "credential_source": frozenset({"none", "environment", "stored"}), + "error_code": frozenset( + {"none", "adapter_error", "parse_error", "evaluator_error", "internal_error"} + ), + } +) + + +@dataclass(frozen=True) +class _AuditPayloadContract: + event_fields: Mapping[str, frozenset[str]] + nullable_fields: Mapping[str, frozenset[str]] + integer_fields: frozenset[str] + decimal_fields: frozenset[str] + boolean_fields: frozenset[str] + hash_fields: frozenset[str] + time_fields: frozenset[str] + enum_values: Mapping[str, frozenset[str]] + + +_ARCHIVE_V1_PAYLOAD_CONTRACT = _AuditPayloadContract( + event_fields=_ARCHIVE_V1_EVENT_PAYLOAD_FIELDS, + nullable_fields=_ARCHIVE_V1_NULLABLE_PAYLOAD_FIELDS, + integer_fields=_ARCHIVE_V1_INTEGER_FIELDS, + decimal_fields=_ARCHIVE_V1_DECIMAL_FIELDS, + boolean_fields=_ARCHIVE_V1_BOOLEAN_FIELDS, + hash_fields=_ARCHIVE_V1_HASH_FIELDS, + time_fields=_ARCHIVE_V1_TIME_FIELDS, + enum_values=_ARCHIVE_V1_ENUM_VALUES, +) + +# The live append/read contract is deliberately a separate copy. A future +# schema revision can extend it while archive-v1 continues to use the frozen +# values above (or introduces a new archive schema and dispatcher). +_EVENT_PAYLOAD_FIELDS = dict(_ARCHIVE_V1_EVENT_PAYLOAD_FIELDS) +_NULLABLE_PAYLOAD_FIELDS = dict(_ARCHIVE_V1_NULLABLE_PAYLOAD_FIELDS) +_INTEGER_FIELDS = frozenset(_ARCHIVE_V1_INTEGER_FIELDS) +_DECIMAL_FIELDS = frozenset(_ARCHIVE_V1_DECIMAL_FIELDS) +_BOOLEAN_FIELDS = frozenset(_ARCHIVE_V1_BOOLEAN_FIELDS) +_HASH_FIELDS = frozenset(_ARCHIVE_V1_HASH_FIELDS) +_TIME_FIELDS = frozenset(_ARCHIVE_V1_TIME_FIELDS) +_ENUM_VALUES = dict(_ARCHIVE_V1_ENUM_VALUES) +_LIVE_PAYLOAD_CONTRACT = _AuditPayloadContract( + event_fields=_EVENT_PAYLOAD_FIELDS, + nullable_fields=_NULLABLE_PAYLOAD_FIELDS, + integer_fields=_INTEGER_FIELDS, + decimal_fields=_DECIMAL_FIELDS, + boolean_fields=_BOOLEAN_FIELDS, + hash_fields=_HASH_FIELDS, + time_fields=_TIME_FIELDS, + enum_values=_ENUM_VALUES, +) + + +def _normalize_payload( + event_type: str, + payload: dict[str, Any] | None, + *, + contract: _AuditPayloadContract = _LIVE_PAYLOAD_CONTRACT, +) -> dict[str, Any]: + allowed = contract.event_fields.get(event_type) if allowed is None: raise ValueError("unsupported audit event type") values = dict(payload or {}) @@ -160,10 +277,10 @@ def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[ normalized: dict[str, Any] = {} for key, value in values.items(): if value is None: - if key not in _NULLABLE_PAYLOAD_FIELDS.get(event_type, frozenset()): + if key not in contract.nullable_fields.get(event_type, frozenset()): raise ValueError(f"audit payload field {key} cannot be null") normalized[key] = None - elif key in _INTEGER_FIELDS: + elif key in contract.integer_fields: if ( isinstance(value, bool) or not isinstance(value, int) @@ -171,7 +288,7 @@ def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[ ): raise ValueError(f"audit payload field {key} must be a non-negative integer") normalized[key] = value - elif key in _DECIMAL_FIELDS: + elif key in contract.decimal_fields: decimal_value = Decimal(str(value)) rendered = format(decimal_value, "f") if ( @@ -181,15 +298,15 @@ def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[ ): raise ValueError(f"audit payload field {key} must be a bounded USD decimal") normalized[key] = rendered - elif key in _BOOLEAN_FIELDS: + elif key in contract.boolean_fields: if not isinstance(value, bool): raise ValueError(f"audit payload field {key} must be boolean") normalized[key] = value - elif key in _HASH_FIELDS: + elif key in contract.hash_fields: if not isinstance(value, str) or not _SAFE_HASH.fullmatch(value): raise ValueError(f"audit payload field {key} must be a SHA-256 hex digest") normalized[key] = value - elif key in _TIME_FIELDS: + elif key in contract.time_fields: if isinstance(value, datetime): timestamp = value elif isinstance(value, str): @@ -210,7 +327,7 @@ def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[ or normalize_provider_metadata(value, max_length=128) is None ): raise ValueError(f"audit payload field {key} must be a short safe token") - allowed_values = _ENUM_VALUES.get(key) + allowed_values = contract.enum_values.get(key) if allowed_values is not None and value not in allowed_values: raise ValueError(f"audit payload field {key} is not an allowed value") normalized[key] = value @@ -218,7 +335,13 @@ def _normalize_payload(event_type: str, payload: dict[str, Any] | None) -> dict[ def _canonical_payload(payload: dict[str, Any]) -> tuple[str, str]: - encoded = json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + encoded = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) return encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest() @@ -249,26 +372,28 @@ def _event_matches(event: AuditEvent, values: dict[str, Any]) -> bool: ) -def validate_audit_payload_for_read( +def _validate_audit_payload_for_read( event_type: object, payload: object, payload_hash: object, + *, + contract: _AuditPayloadContract, ) -> dict[str, Any]: - """Validate one retained payload before it crosses a read boundary. - - Rows normally reach the table only through :func:`append_audit_event`, but - this read-side check fails closed if an import, manual mutation, or storage - corruption introduced arbitrary text. The error never incorporates the - rejected value. - """ - try: if not isinstance(event_type, str) or not isinstance(payload, dict): raise ValueError("invalid audit event shape") - normalized = _normalize_payload(event_type, payload) - _encoded, expected_hash = _canonical_payload(normalized) + normalized = _normalize_payload(event_type, payload, contract=contract) + encoded, expected_hash = _canonical_payload(normalized) + raw_encoded = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) if ( - not isinstance(payload_hash, str) + raw_encoded != encoded + or not isinstance(payload_hash, str) or _SAFE_HASH.fullmatch(payload_hash) is None or not hmac.compare_digest(expected_hash, payload_hash) ): @@ -278,6 +403,42 @@ def validate_audit_payload_for_read( return normalized +def validate_audit_payload_for_read( + event_type: object, + payload: object, + payload_hash: object, +) -> dict[str, Any]: + """Validate one canonical retained payload before a live read boundary. + + Rows normally reach the table only through :func:`append_audit_event`, but + this read-side check fails closed if an import, manual mutation, or storage + corruption introduced a non-canonical representation or arbitrary text. + The error never incorporates the rejected value. + """ + + return _validate_audit_payload_for_read( + event_type, + payload, + payload_hash, + contract=_LIVE_PAYLOAD_CONTRACT, + ) + + +def _validate_audit_archive_v1_payload_for_read( + event_type: object, + payload: object, + payload_hash: object, +) -> dict[str, Any]: + """Validate a payload against the immutable archive-v1 event contract.""" + + return _validate_audit_payload_for_read( + event_type, + payload, + payload_hash, + contract=_ARCHIVE_V1_PAYLOAD_CONTRACT, + ) + + def validate_audit_identity_for_read( name: str, value: object, @@ -300,7 +461,7 @@ def _safe_identity(name: str, value: str | None, *, maximum: int) -> str | None: if not isinstance(value, str) or not value or len(value) > maximum: raise ValueError(f"{name} must contain 1 to {maximum} characters") if ( - not _SAFE_TOKEN.fullmatch(value) + not _SAFE_IDENTITY_TOKEN.fullmatch(value) or "://" in value or normalize_provider_metadata(value, max_length=maximum) is None ): @@ -308,6 +469,248 @@ def _safe_identity(name: str, value: str | None, *, maximum: int) -> str | None: return value +def _validated_utc_timestamp(value: object) -> datetime: + if not isinstance(value, datetime): + raise ValueError("invalid audit timestamp") + if value.tzinfo is None: + # UTCDateTime restores SQLite values as aware UTC. Treat a naive value + # from a compatible driver as UTC rather than letting host local time + # influence retained evidence. + return value.replace(tzinfo=UTC) + if value.utcoffset() != timedelta(0): + return value.astimezone(UTC) + return value + + +_ARCHIVE_V1_RETENTION_DAYS: Mapping[str, int] = MappingProxyType( + {AuditRetentionClass.OPERATIONAL.value: 90, AuditRetentionClass.SECURITY.value: 365} +) +_LIVE_RETENTION_DAYS = dict(_ARCHIVE_V1_RETENTION_DAYS) + + +def _validate_audit_event_values_for_read( + *, + id: object, + event_key: object, + event_type: object, + payload_hash: object, + payload: object, + retention_class: object, + occurred_at: object, + expires_at: object, + correlation_id: object, + run_id: object, + model_id: object, + question_id: object, + worker_id: object, + reservation_id: object, + attempt: object, + provider_attempt: object, + lease_token: object, + duration_ms: object, + payload_validator: Callable[[object, object, object], dict[str, Any]], + retention_days: Mapping[str, int], +) -> AuditEventReadFacts: + try: + normalized_payload = payload_validator(event_type, payload, payload_hash) + validated_id = validate_audit_identity_for_read("id", id, maximum=36) + validated_key = validate_audit_identity_for_read("event_key", event_key, maximum=255) + if validated_id is None or validated_key is None or not isinstance(event_type, str): + raise ValueError("required audit identity is missing") + if not isinstance(payload_hash, str): + raise ValueError("invalid payload hash") + if not isinstance(retention_class, AuditRetentionClass): + raise ValueError("invalid audit retention class") + minimum_days = retention_days.get(retention_class.value) + if minimum_days is None: + raise ValueError("unsupported audit retention class") + + occurred = _validated_utc_timestamp(occurred_at) + expires = _validated_utc_timestamp(expires_at) + minimum_retention = timedelta(days=minimum_days) + if expires - occurred < minimum_retention: + raise ValueError("invalid audit retention interval") + + if isinstance(attempt, bool) or ( + attempt is not None and (not isinstance(attempt, int) or not 0 <= attempt <= _INT32_MAX) + ): + raise ValueError("invalid audit attempt") + if isinstance(provider_attempt, bool) or ( + provider_attempt is not None + and (not isinstance(provider_attempt, int) or not 1 <= provider_attempt <= _INT32_MAX) + ): + raise ValueError("invalid audit provider attempt") + if isinstance(lease_token, bool) or ( + lease_token is not None + and (not isinstance(lease_token, int) or not 0 <= lease_token <= _INT64_MAX) + ): + raise ValueError("invalid audit lease token") + if isinstance(duration_ms, bool) or ( + duration_ms is not None + and ( + not isinstance(duration_ms, (int, float)) + or not math.isfinite(duration_ms) + or duration_ms < 0 + ) + ): + raise ValueError("invalid audit duration") + + duration = None if duration_ms is None else float(duration_ms) + if duration == 0: + duration = 0.0 + return AuditEventReadFacts( + id=validated_id, + event_key=validated_key, + event_type=event_type, + payload_hash=payload_hash, + payload=normalized_payload, + retention_class=retention_class, + occurred_at=occurred, + expires_at=expires, + correlation_id=validate_audit_identity_for_read( + "correlation_id", correlation_id, maximum=128 + ), + run_id=validate_audit_identity_for_read("run_id", run_id, maximum=36), + model_id=validate_audit_identity_for_read("model_id", model_id, maximum=36), + question_id=validate_audit_identity_for_read("question_id", question_id, maximum=36), + worker_id=validate_audit_identity_for_read("worker_id", worker_id, maximum=128), + reservation_id=validate_audit_identity_for_read( + "reservation_id", reservation_id, maximum=36 + ), + attempt=attempt, + provider_attempt=provider_attempt, + lease_token=lease_token, + duration_ms=duration, + ) + except AuditIntegrityError: + raise + except (OverflowError, TypeError, ValueError): + raise AuditIntegrityError("retained audit event failed integrity validation") from None + + +def validate_audit_event_values_for_read( + *, + id: object, + event_key: object, + event_type: object, + payload_hash: object, + payload: object, + retention_class: object, + occurred_at: object, + expires_at: object, + correlation_id: object, + run_id: object, + model_id: object, + question_id: object, + worker_id: object, + reservation_id: object, + attempt: object, + provider_attempt: object, + lease_token: object, + duration_ms: object, +) -> AuditEventReadFacts: + """Validate every storage field before retained evidence is consumed. + + The returned projection is also the exact comparison contract used by the + live application. Rejected values are never included in the raised error. + """ + + return _validate_audit_event_values_for_read( + id=id, + event_key=event_key, + event_type=event_type, + payload_hash=payload_hash, + payload=payload, + retention_class=retention_class, + occurred_at=occurred_at, + expires_at=expires_at, + correlation_id=correlation_id, + run_id=run_id, + model_id=model_id, + question_id=question_id, + worker_id=worker_id, + reservation_id=reservation_id, + attempt=attempt, + provider_attempt=provider_attempt, + lease_token=lease_token, + duration_ms=duration_ms, + payload_validator=validate_audit_payload_for_read, + retention_days=_LIVE_RETENTION_DAYS, + ) + + +def validate_audit_archive_v1_event_values_for_read( + *, + id: object, + event_key: object, + event_type: object, + payload_hash: object, + payload: object, + retention_class: object, + occurred_at: object, + expires_at: object, + correlation_id: object, + run_id: object, + model_id: object, + question_id: object, + worker_id: object, + reservation_id: object, + attempt: object, + provider_attempt: object, + lease_token: object, + duration_ms: object, +) -> AuditEventReadFacts: + """Validate complete storage facts against the frozen archive-v1 contract.""" + + return _validate_audit_event_values_for_read( + id=id, + event_key=event_key, + event_type=event_type, + payload_hash=payload_hash, + payload=payload, + retention_class=retention_class, + occurred_at=occurred_at, + expires_at=expires_at, + correlation_id=correlation_id, + run_id=run_id, + model_id=model_id, + question_id=question_id, + worker_id=worker_id, + reservation_id=reservation_id, + attempt=attempt, + provider_attempt=provider_attempt, + lease_token=lease_token, + duration_ms=duration_ms, + payload_validator=_validate_audit_archive_v1_payload_for_read, + retention_days=_ARCHIVE_V1_RETENTION_DAYS, + ) + + +def validate_audit_event_for_read(event: AuditEvent) -> AuditEventReadFacts: + """Return the complete validated storage projection for one ORM row.""" + + return validate_audit_event_values_for_read( + id=event.id, + event_key=event.event_key, + event_type=event.event_type, + payload_hash=event.payload_hash, + payload=event.payload, + retention_class=event.retention_class, + occurred_at=event.occurred_at, + expires_at=event.expires_at, + correlation_id=event.correlation_id, + run_id=event.run_id, + model_id=event.model_id, + question_id=event.question_id, + worker_id=event.worker_id, + reservation_id=event.reservation_id, + attempt=event.attempt, + provider_attempt=event.provider_attempt, + lease_token=event.lease_token, + duration_ms=event.duration_ms, + ) + + def append_audit_event( session: Session, *, diff --git a/backend/app/governance/audit_archive.py b/backend/app/governance/audit_archive.py new file mode 100644 index 0000000..e7bcdea --- /dev/null +++ b/backend/app/governance/audit_archive.py @@ -0,0 +1,669 @@ +"""Canonical, bounded audit-retention archive format. + +The archive is an internal sensitive operations artifact. Its hashes detect +accidental or uncoordinated changes; they are not signatures and do not make a +regular file immutable. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import os +import re +import secrets +import stat +from collections import Counter +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from app.governance.audit import ( + AuditEventReadFacts, + AuditIntegrityError, + validate_audit_archive_v1_event_values_for_read, +) +from app.models import AuditRetentionClass + +ARCHIVE_SCHEMA = "llmbenchlab-audit-archive-v1" +ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS = frozenset({"20260828_0005"}) +ARCHIVE_V1_RETENTION_VALUES = ("operational", "security") +ARCHIVE_EVENT_LIMIT = 10_000 +ARCHIVE_FILE_LIMIT_BYTES = 128 * 1024 * 1024 +ARCHIVE_LINE_LIMIT_BYTES = 64 * 1024 +_CONTENT_HASH_DOMAIN = b"LLMBenchLab audit archive content v1\x00" +_SHA256 = re.compile(r"[a-f0-9]{64}") +_ALEMBIC_REVISION = re.compile(r"[A-Za-z0-9_]{1,64}") + +_HEADER_KEYS = frozenset( + { + "record_type", + "schema", + "cutoff_at", + "source_alembic_head", + "event_limit", + } +) +_EVENT_KEYS = frozenset( + { + "record_type", + "id", + "event_key", + "event_type", + "payload_hash", + "payload", + "retention_class", + "occurred_at", + "expires_at", + "correlation_id", + "run_id", + "model_id", + "question_id", + "worker_id", + "reservation_id", + "attempt", + "provider_attempt", + "lease_token", + "duration_ms_hex", + } +) +_MANIFEST_KEYS = frozenset( + { + "record_type", + "schema", + "cutoff_at", + "source_alembic_head", + "event_count", + "event_type_counts", + "retention_class_counts", + "occurred_at_min", + "occurred_at_max", + "expires_at_min", + "expires_at_max", + "has_more_eligible", + "content_sha256", + } +) + + +class AuditArchiveError(RuntimeError): + """A fixed-code archive format, path, or integrity failure.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class VerifiedAuditArchive: + """Strictly parsed archive held in memory after single-descriptor verification.""" + + cutoff_at: datetime + source_alembic_head: str + events: tuple[AuditEventReadFacts, ...] + has_more_eligible: bool + content_sha256: str + archive_sha256: str + + +@dataclass(frozen=True) +class WrittenAuditArchive: + """Archive output summary safe for fixed CLI reporting.""" + + event_count: int + has_more_eligible: bool + content_sha256: str + archive_sha256: str + + +def _canonical_json(value: dict[str, Any]) -> bytes: + try: + encoded = json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError): + raise AuditArchiveError("archive_value_not_canonical_json") from None + if len(encoded) > ARCHIVE_LINE_LIMIT_BYTES: + raise AuditArchiveError("archive_line_limit_exceeded") + return encoded + + +def _format_timestamp(value: datetime) -> str: + value = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _parse_timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise AuditArchiveError("archive_timestamp_invalid") + try: + parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=UTC) + except ValueError: + raise AuditArchiveError("archive_timestamp_invalid") from None + if _format_timestamp(parsed) != value: + raise AuditArchiveError("archive_timestamp_not_canonical") + return parsed + + +def _duration_to_hex(value: float | None) -> str | None: + if value is None: + return None + normalized = float(value) + if not math.isfinite(normalized) or normalized < 0: + raise AuditArchiveError("archive_duration_invalid") + if normalized == 0: + normalized = 0.0 + return normalized.hex() + + +def _duration_from_hex(value: object) -> float | None: + if value is None: + return None + if not isinstance(value, str): + raise AuditArchiveError("archive_duration_invalid") + try: + parsed = float.fromhex(value) + except ValueError: + raise AuditArchiveError("archive_duration_invalid") from None + canonical = _duration_to_hex(parsed) + if canonical != value: + raise AuditArchiveError("archive_duration_not_canonical") + return parsed + + +def _event_record(event: AuditEventReadFacts) -> dict[str, Any]: + return { + "record_type": "audit_event", + "id": event.id, + "event_key": event.event_key, + "event_type": event.event_type, + "payload_hash": event.payload_hash, + "payload": dict(event.payload), + "retention_class": event.retention_class.value, + "occurred_at": _format_timestamp(event.occurred_at), + "expires_at": _format_timestamp(event.expires_at), + "correlation_id": event.correlation_id, + "run_id": event.run_id, + "model_id": event.model_id, + "question_id": event.question_id, + "worker_id": event.worker_id, + "reservation_id": event.reservation_id, + "attempt": event.attempt, + "provider_attempt": event.provider_attempt, + "lease_token": event.lease_token, + "duration_ms_hex": _duration_to_hex(event.duration_ms), + } + + +def _validated_facts(event: AuditEventReadFacts) -> AuditEventReadFacts: + try: + return validate_audit_archive_v1_event_values_for_read( + id=event.id, + event_key=event.event_key, + event_type=event.event_type, + payload_hash=event.payload_hash, + payload=event.payload, + retention_class=event.retention_class, + occurred_at=event.occurred_at, + expires_at=event.expires_at, + correlation_id=event.correlation_id, + run_id=event.run_id, + model_id=event.model_id, + question_id=event.question_id, + worker_id=event.worker_id, + reservation_id=event.reservation_id, + attempt=event.attempt, + provider_attempt=event.provider_attempt, + lease_token=event.lease_token, + duration_ms=event.duration_ms, + ) + except (AttributeError, AuditIntegrityError): + raise AuditArchiveError("archive_event_integrity_invalid") from None + + +def _content_digest(lines: list[bytes]) -> str: + digest = hashlib.sha256() + digest.update(_CONTENT_HASH_DOMAIN) + for line in lines: + digest.update(len(line).to_bytes(8, "big")) + digest.update(line) + return digest.hexdigest() + + +def _validate_source_head(value: object) -> str: + if not isinstance(value, str) or not _ALEMBIC_REVISION.fullmatch(value): + raise AuditArchiveError("archive_source_revision_invalid") + if value not in ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS: + raise AuditArchiveError("archive_source_revision_unsupported") + return value + + +def build_archive_bytes( + events: tuple[AuditEventReadFacts, ...], + *, + cutoff_at: datetime, + source_alembic_head: str, + has_more_eligible: bool, +) -> tuple[bytes, str]: + """Build canonical archive bytes and the non-self-referential content digest.""" + + if len(events) > ARCHIVE_EVENT_LIMIT: + raise AuditArchiveError("archive_event_limit_exceeded") + if not isinstance(has_more_eligible, bool): + raise AuditArchiveError("archive_has_more_invalid") + source_alembic_head = _validate_source_head(source_alembic_head) + try: + cutoff = _parse_timestamp(_format_timestamp(cutoff_at)) + except (AttributeError, OverflowError, ValueError): + raise AuditArchiveError("archive_cutoff_invalid") from None + events = tuple(_validated_facts(event) for event in events) + + previous_order: tuple[datetime, str] | None = None + seen_ids: set[str] = set() + seen_keys: set[str] = set() + for event in events: + order = (event.expires_at, event.id) + if previous_order is not None and order <= previous_order: + raise AuditArchiveError("archive_events_not_strictly_ordered") + if event.id in seen_ids or event.event_key in seen_keys: + raise AuditArchiveError("archive_event_identity_duplicated") + if event.expires_at >= cutoff: + raise AuditArchiveError("archive_event_not_eligible") + previous_order = order + seen_ids.add(event.id) + seen_keys.add(event.event_key) + + header = { + "record_type": "header", + "schema": ARCHIVE_SCHEMA, + "cutoff_at": _format_timestamp(cutoff), + "source_alembic_head": source_alembic_head, + "event_limit": ARCHIVE_EVENT_LIMIT, + } + content_lines = [_canonical_json(header), *(_canonical_json(_event_record(e)) for e in events)] + content_sha256 = _content_digest(content_lines) + + event_type_counts = dict(sorted(Counter(event.event_type for event in events).items())) + retention_counts = { + retention: sum(event.retention_class.value == retention for event in events) + for retention in ARCHIVE_V1_RETENTION_VALUES + } + occurred_values = [event.occurred_at for event in events] + expiry_values = [event.expires_at for event in events] + manifest = { + "record_type": "manifest", + "schema": ARCHIVE_SCHEMA, + "cutoff_at": _format_timestamp(cutoff), + "source_alembic_head": source_alembic_head, + "event_count": len(events), + "event_type_counts": event_type_counts, + "retention_class_counts": retention_counts, + "occurred_at_min": _format_timestamp(min(occurred_values)) if events else None, + "occurred_at_max": _format_timestamp(max(occurred_values)) if events else None, + "expires_at_min": _format_timestamp(min(expiry_values)) if events else None, + "expires_at_max": _format_timestamp(max(expiry_values)) if events else None, + "has_more_eligible": has_more_eligible, + "content_sha256": content_sha256, + } + all_lines = [*content_lines, _canonical_json(manifest)] + data = b"\n".join(all_lines) + b"\n" + if len(data) > ARCHIVE_FILE_LIMIT_BYTES: + raise AuditArchiveError("archive_file_limit_exceeded") + return data, content_sha256 + + +def _write_all(file_descriptor: int, data: bytes) -> None: + position = 0 + while position < len(data): + written = os.write(file_descriptor, data[position:]) + if written <= 0: + raise AuditArchiveError("archive_output_write_failed") + position += written + + +def write_archive_no_replace(path: Path, data: bytes) -> str: + """Install one 0600 archive atomically without replacing any target.""" + + output = Path(path) + parent = output.parent + if not output.name: + raise AuditArchiveError("archive_output_parent_invalid") + if len(data) > ARCHIVE_FILE_LIMIT_BYTES: + raise AuditArchiveError("archive_file_limit_exceeded") + + nofollow = getattr(os, "O_NOFOLLOW", 0) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | nofollow + directory_fd: int | None = None + temporary_name = f".{output.name}.tmp-{secrets.token_hex(12)}" + file_descriptor: int | None = None + installed = False + completed = False + try: + try: + directory_fd = os.open(parent, directory_flags) + except OSError: + raise AuditArchiveError("archive_output_parent_invalid") from None + parent_details = os.fstat(directory_fd) + if not stat.S_ISDIR(parent_details.st_mode) or parent_details.st_uid != os.geteuid(): + raise AuditArchiveError("archive_output_parent_not_owned") + if stat.S_IMODE(parent_details.st_mode) & 0o022: + raise AuditArchiveError("archive_output_parent_permissions_too_broad") + file_descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow, + 0o600, + dir_fd=directory_fd, + ) + os.fchmod(file_descriptor, 0o600) + _write_all(file_descriptor, data) + os.fsync(file_descriptor) + os.close(file_descriptor) + file_descriptor = None + os.link( + temporary_name, + output.name, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + follow_symlinks=False, + ) + installed = True + os.unlink(temporary_name, dir_fd=directory_fd) + os.fsync(directory_fd) + completed = True + except FileExistsError: + raise AuditArchiveError("archive_output_exists") from None + except AuditArchiveError: + raise + except OSError: + raise AuditArchiveError("archive_output_filesystem_error") from None + finally: + if file_descriptor is not None: + with suppress(OSError): + os.close(file_descriptor) + if directory_fd is not None: + with suppress(OSError): + os.unlink(temporary_name, dir_fd=directory_fd) + if installed and not completed: + # The target can only be ours after a successful no-replace + # link. Remove it when the directory durability step fails so + # a failed command does not leave an apparent success artifact. + with suppress(OSError): + os.unlink(output.name, dir_fd=directory_fd) + with suppress(OSError): + os.fsync(directory_fd) + with suppress(OSError): + os.close(directory_fd) + return hashlib.sha256(data).hexdigest() + + +def write_archive( + path: Path, + events: tuple[AuditEventReadFacts, ...], + *, + cutoff_at: datetime, + source_alembic_head: str, + has_more_eligible: bool, +) -> WrittenAuditArchive: + data, content_sha256 = build_archive_bytes( + events, + cutoff_at=cutoff_at, + source_alembic_head=source_alembic_head, + has_more_eligible=has_more_eligible, + ) + archive_sha256 = write_archive_no_replace(path, data) + return WrittenAuditArchive( + event_count=len(events), + has_more_eligible=has_more_eligible, + content_sha256=content_sha256, + archive_sha256=archive_sha256, + ) + + +def _reject_json_constant(_value: str) -> None: + raise AuditArchiveError("archive_json_non_finite") + + +def _strict_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise AuditArchiveError("archive_json_duplicate_key") + result[key] = value + return result + + +def _decode_line(line: bytes) -> dict[str, Any]: + if not line or len(line) > ARCHIVE_LINE_LIMIT_BYTES: + raise AuditArchiveError("archive_line_invalid") + try: + text = line.decode("utf-8") + value = json.loads( + text, + object_pairs_hook=_strict_object, + parse_constant=_reject_json_constant, + ) + except AuditArchiveError: + raise + except (UnicodeDecodeError, json.JSONDecodeError): + raise AuditArchiveError("archive_json_invalid") from None + if not isinstance(value, dict): + raise AuditArchiveError("archive_record_not_object") + if _canonical_json(value) != line: + raise AuditArchiveError("archive_record_not_canonical") + return value + + +def _require_exact_keys(value: dict[str, Any], keys: frozenset[str], code: str) -> None: + if frozenset(value) != keys: + raise AuditArchiveError(code) + + +def _parse_event(value: dict[str, Any]) -> AuditEventReadFacts: + _require_exact_keys(value, _EVENT_KEYS, "archive_event_schema_invalid") + if value["record_type"] != "audit_event": + raise AuditArchiveError("archive_event_type_invalid") + retention_value = value["retention_class"] + if not isinstance(retention_value, str) or retention_value not in ARCHIVE_V1_RETENTION_VALUES: + raise AuditArchiveError("archive_retention_class_invalid") from None + try: + retention = AuditRetentionClass(retention_value) + except ValueError: + raise AuditArchiveError("archive_retention_class_invalid") from None + try: + return validate_audit_archive_v1_event_values_for_read( + id=value["id"], + event_key=value["event_key"], + event_type=value["event_type"], + payload_hash=value["payload_hash"], + payload=value["payload"], + retention_class=retention, + occurred_at=_parse_timestamp(value["occurred_at"]), + expires_at=_parse_timestamp(value["expires_at"]), + correlation_id=value["correlation_id"], + run_id=value["run_id"], + model_id=value["model_id"], + question_id=value["question_id"], + worker_id=value["worker_id"], + reservation_id=value["reservation_id"], + attempt=value["attempt"], + provider_attempt=value["provider_attempt"], + lease_token=value["lease_token"], + duration_ms=_duration_from_hex(value["duration_ms_hex"]), + ) + except AuditIntegrityError: + raise AuditArchiveError("archive_event_integrity_invalid") from None + + +def _read_owned_private_file(path: Path) -> bytes: + nofollow = getattr(os, "O_NOFOLLOW", 0) + nonblock = getattr(os, "O_NONBLOCK", 0) + try: + descriptor = os.open(Path(path), os.O_RDONLY | nofollow | nonblock) + except OSError: + raise AuditArchiveError("archive_input_open_failed") from None + try: + details = os.fstat(descriptor) + if not stat.S_ISREG(details.st_mode): + raise AuditArchiveError("archive_input_not_regular") + if details.st_uid != os.geteuid(): + raise AuditArchiveError("archive_input_wrong_owner") + if stat.S_IMODE(details.st_mode) & ~0o600: + raise AuditArchiveError("archive_input_permissions_too_broad") + if details.st_size > ARCHIVE_FILE_LIMIT_BYTES: + raise AuditArchiveError("archive_file_limit_exceeded") + chunks: list[bytes] = [] + remaining = ARCHIVE_FILE_LIMIT_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) > ARCHIVE_FILE_LIMIT_BYTES: + raise AuditArchiveError("archive_file_limit_exceeded") + return data + except OSError: + raise AuditArchiveError("archive_input_read_failed") from None + finally: + with suppress(OSError): + os.close(descriptor) + + +def _validate_digest(value: object, *, code: str) -> str: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise AuditArchiveError(code) + return value + + +def _validate_count_mapping(value: object) -> None: + if not isinstance(value, dict) or any( + not isinstance(key, str) + or isinstance(count, bool) + or not isinstance(count, int) + or count < 0 + for key, count in value.items() + ): + raise AuditArchiveError("archive_manifest_rollup_invalid") + + +def verify_archive( + path: Path, + *, + expected_sha256: str | None = None, +) -> VerifiedAuditArchive: + """Strictly verify one archive from a single opened file descriptor.""" + + data = _read_owned_private_file(path) + archive_sha256 = hashlib.sha256(data).hexdigest() + if expected_sha256 is not None: + expected = _validate_digest(expected_sha256, code="archive_expected_digest_invalid") + if not hmac.compare_digest(archive_sha256, expected): + raise AuditArchiveError("archive_digest_mismatch") + if not data or not data.endswith(b"\n") or data.endswith(b"\n\n"): + raise AuditArchiveError("archive_line_termination_invalid") + # One header, at most ARCHIVE_EVENT_LIMIT events, and one manifest are the + # complete v1 record set. Enforce that bound before splitting or decoding + # attacker-controlled small lines into Python objects. + if data.count(b"\n") > ARCHIVE_EVENT_LIMIT + 2: + raise AuditArchiveError("archive_event_limit_exceeded") + raw_lines = data[:-1].split(b"\n") + if len(raw_lines) < 2 or any(not line for line in raw_lines): + raise AuditArchiveError("archive_record_sequence_invalid") + + values = [_decode_line(line) for line in raw_lines] + header = values[0] + manifest = values[-1] + _require_exact_keys(header, _HEADER_KEYS, "archive_header_schema_invalid") + _require_exact_keys(manifest, _MANIFEST_KEYS, "archive_manifest_schema_invalid") + if header["record_type"] != "header" or manifest["record_type"] != "manifest": + raise AuditArchiveError("archive_record_sequence_invalid") + if header["schema"] != ARCHIVE_SCHEMA or manifest["schema"] != ARCHIVE_SCHEMA: + raise AuditArchiveError("archive_schema_unsupported") + if header["event_limit"] != ARCHIVE_EVENT_LIMIT: + raise AuditArchiveError("archive_event_limit_invalid") + cutoff_at = _parse_timestamp(header["cutoff_at"]) + source_head = _validate_source_head(header["source_alembic_head"]) + if manifest["cutoff_at"] != header["cutoff_at"]: + raise AuditArchiveError("archive_manifest_header_mismatch") + if manifest["source_alembic_head"] != source_head: + raise AuditArchiveError("archive_manifest_header_mismatch") + if ( + isinstance(manifest["event_count"], bool) + or not isinstance(manifest["event_count"], int) + or not 0 <= manifest["event_count"] <= ARCHIVE_EVENT_LIMIT + ): + raise AuditArchiveError("archive_manifest_rollup_invalid") + _validate_count_mapping(manifest["event_type_counts"]) + _validate_count_mapping(manifest["retention_class_counts"]) + + event_values = values[1:-1] + if len(event_values) > ARCHIVE_EVENT_LIMIT: + raise AuditArchiveError("archive_event_limit_exceeded") + events = tuple(_parse_event(value) for value in event_values) + previous_order: tuple[datetime, str] | None = None + seen_ids: set[str] = set() + seen_keys: set[str] = set() + for event in events: + order = (event.expires_at, event.id) + if previous_order is not None and order <= previous_order: + raise AuditArchiveError("archive_events_not_strictly_ordered") + if event.id in seen_ids or event.event_key in seen_keys: + raise AuditArchiveError("archive_event_identity_duplicated") + if event.expires_at >= cutoff_at: + raise AuditArchiveError("archive_event_not_eligible") + previous_order = order + seen_ids.add(event.id) + seen_keys.add(event.event_key) + + expected_content = _content_digest(raw_lines[:-1]) + content_sha256 = _validate_digest( + manifest["content_sha256"], code="archive_content_digest_invalid" + ) + if not hmac.compare_digest(expected_content, content_sha256): + raise AuditArchiveError("archive_content_digest_mismatch") + + expected_type_counts = dict(sorted(Counter(event.event_type for event in events).items())) + expected_retention_counts = { + retention: sum(event.retention_class.value == retention for event in events) + for retention in ARCHIVE_V1_RETENTION_VALUES + } + occurred = [event.occurred_at for event in events] + expires = [event.expires_at for event in events] + expected_rollup = { + "event_count": len(events), + "event_type_counts": expected_type_counts, + "retention_class_counts": expected_retention_counts, + "occurred_at_min": _format_timestamp(min(occurred)) if events else None, + "occurred_at_max": _format_timestamp(max(occurred)) if events else None, + "expires_at_min": _format_timestamp(min(expires)) if events else None, + "expires_at_max": _format_timestamp(max(expires)) if events else None, + } + for key, expected_value in expected_rollup.items(): + if manifest[key] != expected_value: + raise AuditArchiveError("archive_manifest_rollup_mismatch") + if not isinstance(manifest["has_more_eligible"], bool): + raise AuditArchiveError("archive_has_more_invalid") + + return VerifiedAuditArchive( + cutoff_at=cutoff_at, + source_alembic_head=source_head, + events=events, + has_more_eligible=manifest["has_more_eligible"], + content_sha256=content_sha256, + archive_sha256=archive_sha256, + ) + + +def confirm_archive_digest(archive: VerifiedAuditArchive, confirmation: str) -> None: + expected = _validate_digest(confirmation, code="archive_confirmation_digest_invalid") + if not hmac.compare_digest(archive.archive_sha256, expected): + raise AuditArchiveError("archive_confirmation_digest_mismatch") diff --git a/backend/app/governance/audit_retention.py b/backend/app/governance/audit_retention.py new file mode 100644 index 0000000..fd724d4 --- /dev/null +++ b/backend/app/governance/audit_retention.py @@ -0,0 +1,469 @@ +"""Database-authoritative audit archive, reconcile, restore, and exact delete operations.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from sqlalchemy import delete, or_, select +from sqlalchemy.orm import Session, sessionmaker + +from app.db.clock import database_utc_now +from app.db.prepare_migrations import database_heads, expected_database_heads +from app.governance.audit import ( + AuditEventReadFacts, + AuditIntegrityError, + validate_audit_event_for_read, +) +from app.governance.audit_archive import ( + ARCHIVE_EVENT_LIMIT, + ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS, + VerifiedAuditArchive, + WrittenAuditArchive, + write_archive, +) +from app.governance.audit_retention_errors import ( + AuditRetentionCommitOutcomeUnknownError, + AuditRetentionCommittedVerificationError, + AuditRetentionError, +) +from app.models import AuditEvent, ProviderCallReservation + +_POSTGRES_ADVISORY_LOCK = (1280068930, 1381256526) +_QUERY_CHUNK_SIZE = 400 + +ReconcileStatus = Literal[ + "all_exact", + "all_absent", + "mixed_exact_absent", + "conflict", + "empty_archive", +] + + +@dataclass(frozen=True) +class ReconcileReport: + status: ReconcileStatus + event_count: int + exact_count: int + absent_count: int + conflict_count: int + + +@dataclass(frozen=True) +class MutationReport: + status: str + event_count: int + changed_count: int + archive_sha256: str + + +@dataclass +class _CommitState: + confirmed: bool = False + event_count: int = 0 + archive_sha256: str = "" + + +@contextmanager +def _mutation_session( + session_factory: sessionmaker[Session], + state: _CommitState, +) -> Iterator[Session]: + session = session_factory() + try: + yield session + finally: + try: + session.close() + except Exception as exc: + if state.confirmed: + raise AuditRetentionCommittedVerificationError( + "audit_retention_committed_cleanup_failed", + event_count=state.event_count, + archive_sha256=state.archive_sha256, + ) from exc + + +def _configure_read_snapshot(session: Session) -> None: + connection = session.connection() + dialect = connection.dialect.name + if dialect == "sqlite": + connection.exec_driver_sql("BEGIN") + elif dialect == "postgresql": + connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + else: + raise AuditRetentionError("audit_retention_database_dialect_unsupported") + + +def _require_current_head(session: Session) -> str: + current = tuple(sorted(database_heads(session.connection()))) + expected = tuple(sorted(expected_database_heads())) + if current != expected or len(current) != 1: + raise AuditRetentionError("audit_retention_database_schema_not_ready") + if current[0] not in ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS: + raise AuditRetentionError("audit_retention_database_schema_not_compatible") + return current[0] + + +def archive_expired_events( + session_factory: sessionmaker[Session], + output: Path, +) -> WrittenAuditArchive: + """Snapshot currently expired events and atomically write one bounded archive.""" + + with session_factory() as session: + try: + _configure_read_snapshot(session) + source_head = _require_current_head(session) + cutoff_at = database_utc_now(session) + rows = session.scalars( + select(AuditEvent) + .where(AuditEvent.expires_at < cutoff_at) + .order_by(AuditEvent.expires_at, AuditEvent.id) + .limit(ARCHIVE_EVENT_LIMIT + 1) + ).all() + selected = rows[:ARCHIVE_EVENT_LIMIT] + events = tuple(validate_audit_event_for_read(event) for event in selected) + has_more = len(rows) > ARCHIVE_EVENT_LIMIT + except AuditRetentionError: + raise + except AuditIntegrityError: + raise AuditRetentionError("audit_retention_event_integrity_error") from None + except Exception as exc: + raise AuditRetentionError("audit_retention_archive_snapshot_failed") from exc + finally: + session.rollback() + + return write_archive( + output, + events, + cutoff_at=cutoff_at, + source_alembic_head=source_head, + has_more_eligible=has_more, + ) + + +def _configure_mutation_transaction(session: Session) -> None: + connection = session.connection() + dialect = connection.dialect.name + if dialect == "sqlite": + connection.exec_driver_sql("BEGIN IMMEDIATE") + elif dialect == "postgresql": + connection.exec_driver_sql( + f"SELECT pg_advisory_xact_lock({_POSTGRES_ADVISORY_LOCK[0]}, " + f"{_POSTGRES_ADVISORY_LOCK[1]})" + ) + else: + raise AuditRetentionError("audit_retention_database_dialect_unsupported") + + +def _chunks(values: tuple[AuditEventReadFacts, ...]): + for start in range(0, len(values), _QUERY_CHUNK_SIZE): + yield values[start : start + _QUERY_CHUNK_SIZE] + + +def _load_candidate_rows( + session: Session, + events: tuple[AuditEventReadFacts, ...], + *, + for_update: bool, +) -> list[AuditEvent]: + rows_by_identity: dict[str, AuditEvent] = {} + for chunk in _chunks(events): + ids = [event.id for event in chunk] + keys = [event.event_key for event in chunk] + statement = select(AuditEvent).where( + or_(AuditEvent.id.in_(ids), AuditEvent.event_key.in_(keys)) + ) + if for_update and session.get_bind().dialect.name == "postgresql": + statement = statement.with_for_update() + for row in session.scalars(statement): + rows_by_identity[row.id] = row + return list(rows_by_identity.values()) + + +def _classify( + events: tuple[AuditEventReadFacts, ...], + rows: list[AuditEvent], +) -> tuple[ReconcileReport, dict[str, AuditEvent]]: + by_id = {row.id: row for row in rows} + by_key = {row.event_key: row for row in rows} + exact_count = 0 + absent_count = 0 + conflict_count = 0 + exact_rows: dict[str, AuditEvent] = {} + for expected in events: + id_row = by_id.get(expected.id) + key_row = by_key.get(expected.event_key) + if id_row is None and key_row is None: + absent_count += 1 + continue + if id_row is None or key_row is None or id_row.id != key_row.id: + conflict_count += 1 + continue + try: + actual = validate_audit_event_for_read(id_row) + except AuditIntegrityError: + conflict_count += 1 + continue + if actual == expected: + exact_count += 1 + exact_rows[expected.id] = id_row + else: + conflict_count += 1 + + event_count = len(events) + if event_count == 0: + status: ReconcileStatus = "empty_archive" + elif conflict_count: + status = "conflict" + elif exact_count == event_count: + status = "all_exact" + elif absent_count == event_count: + status = "all_absent" + else: + status = "mixed_exact_absent" + return ( + ReconcileReport( + status=status, + event_count=event_count, + exact_count=exact_count, + absent_count=absent_count, + conflict_count=conflict_count, + ), + exact_rows, + ) + + +def _reconcile_in_session( + session: Session, + archive: VerifiedAuditArchive, + *, + for_update: bool, +) -> tuple[ReconcileReport, dict[str, AuditEvent]]: + rows = _load_candidate_rows(session, archive.events, for_update=for_update) + return _classify(archive.events, rows) + + +def reconcile_archive( + session_factory: sessionmaker[Session], + archive: VerifiedAuditArchive, +) -> ReconcileReport: + """Read-only exact reconciliation for commit-unknown recovery.""" + + with session_factory() as session: + try: + _configure_read_snapshot(session) + _require_current_head(session) + report, _rows = _reconcile_in_session(session, archive, for_update=False) + return report + except AuditRetentionError: + raise + except Exception as exc: + raise AuditRetentionError("audit_retention_reconcile_failed") from exc + finally: + session.rollback() + + +def _require_reservations( + session: Session, + events: tuple[AuditEventReadFacts, ...], +) -> None: + reservation_ids = tuple( + sorted({event.reservation_id for event in events if event.reservation_id is not None}) + ) + if not reservation_ids: + return + found: set[str] = set() + for start in range(0, len(reservation_ids), _QUERY_CHUNK_SIZE): + chunk = reservation_ids[start : start + _QUERY_CHUNK_SIZE] + found.update( + session.scalars( + select(ProviderCallReservation.id).where(ProviderCallReservation.id.in_(chunk)) + ) + ) + if found != set(reservation_ids): + raise AuditRetentionError("audit_retention_reservation_missing") + + +def _postcommit_report( + session_factory: sessionmaker[Session], + archive: VerifiedAuditArchive, +) -> ReconcileReport: + try: + return reconcile_archive(session_factory, archive) + except Exception as exc: + raise AuditRetentionCommittedVerificationError( + "audit_retention_postcommit_verification_failed", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) from exc + + +def _delete_exact_event_ids( + session: Session, + events: tuple[AuditEventReadFacts, ...], +) -> int: + """Delete the already-locked exact ID set and return the driver rowcount.""" + + deleted_count = 0 + for chunk in _chunks(events): + result = session.execute( + delete(AuditEvent) + .where(AuditEvent.id.in_([event.id for event in chunk])) + .execution_options(synchronize_session=False) + ) + rowcount = getattr(result, "rowcount", None) + if isinstance(rowcount, bool) or not isinstance(rowcount, int) or rowcount < 0: + raise RuntimeError("audit_retention_delete_rowcount_unavailable") + deleted_count += rowcount + return deleted_count + + +def restore_archive( + session_factory: sessionmaker[Session], + archive: VerifiedAuditArchive, +) -> MutationReport: + """Insert only absent exact rows and reject every conflicting durable fact.""" + + changed_count = 0 + commit_state = _CommitState( + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) + with _mutation_session(session_factory, commit_state) as session: + try: + _configure_mutation_transaction(session) + _require_current_head(session) + _require_reservations(session, archive.events) + report, _exact_rows = _reconcile_in_session(session, archive, for_update=True) + if report.status == "conflict": + raise AuditRetentionError("audit_retention_restore_conflict") + if report.status in {"all_absent", "mixed_exact_absent"}: + rows = _load_candidate_rows(session, archive.events, for_update=False) + existing_ids = {row.id for row in rows} + existing_keys = {row.event_key for row in rows} + missing = [ + event + for event in archive.events + if event.id not in existing_ids and event.event_key not in existing_keys + ] + for event in missing: + session.add(AuditEvent(**event.as_insert_values())) + changed_count = len(missing) + try: + session.flush() + except Exception as exc: + session.rollback() + raise AuditRetentionError("audit_retention_restore_precommit_failed") from exc + try: + session.commit() + except Exception as exc: + raise AuditRetentionCommitOutcomeUnknownError( + "audit_retention_restore_commit_outcome_unknown", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) from exc + commit_state.confirmed = True + else: + session.rollback() + except (AuditRetentionError, AuditRetentionCommitOutcomeUnknownError): + raise + except Exception as exc: + if commit_state.confirmed: + raise AuditRetentionCommittedVerificationError( + "audit_retention_restore_committed_cleanup_failed", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) from exc + session.rollback() + raise AuditRetentionError("audit_retention_restore_precommit_failed") from exc + + postcommit = _postcommit_report(session_factory, archive) + if postcommit.status not in {"all_exact", "empty_archive"}: + raise AuditRetentionCommittedVerificationError( + "audit_retention_restore_postcommit_mismatch", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) + return MutationReport( + status="restored" if changed_count else "already_exact", + event_count=len(archive.events), + changed_count=changed_count, + archive_sha256=archive.archive_sha256, + ) + + +def delete_archive_events( + session_factory: sessionmaker[Session], + archive: VerifiedAuditArchive, +) -> MutationReport: + """Delete only the exact archived id/key set after expiry and full comparison.""" + + changed_count = 0 + commit_state = _CommitState( + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) + with _mutation_session(session_factory, commit_state) as session: + try: + _configure_mutation_transaction(session) + _require_current_head(session) + database_now = database_utc_now(session) + if database_now < archive.cutoff_at or any( + event.expires_at >= database_now for event in archive.events + ): + raise AuditRetentionError("audit_retention_delete_not_expired") + report, _exact_rows = _reconcile_in_session(session, archive, for_update=True) + if report.status in {"conflict", "mixed_exact_absent"}: + raise AuditRetentionError("audit_retention_delete_conflict") + if report.status == "all_exact": + try: + actual_deleted_count = _delete_exact_event_ids(session, archive.events) + except Exception as exc: + session.rollback() + raise AuditRetentionError("audit_retention_delete_precommit_failed") from exc + if actual_deleted_count != len(archive.events): + session.rollback() + raise AuditRetentionError("audit_retention_delete_count_mismatch") + changed_count = actual_deleted_count + try: + session.commit() + except Exception as exc: + raise AuditRetentionCommitOutcomeUnknownError( + "audit_retention_delete_commit_outcome_unknown", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) from exc + commit_state.confirmed = True + else: + session.rollback() + except (AuditRetentionError, AuditRetentionCommitOutcomeUnknownError): + raise + except Exception as exc: + if commit_state.confirmed: + raise AuditRetentionCommittedVerificationError( + "audit_retention_delete_committed_cleanup_failed", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) from exc + session.rollback() + raise AuditRetentionError("audit_retention_delete_precommit_failed") from exc + + postcommit = _postcommit_report(session_factory, archive) + if postcommit.status not in {"all_absent", "empty_archive"}: + raise AuditRetentionCommittedVerificationError( + "audit_retention_delete_postcommit_mismatch", + event_count=len(archive.events), + archive_sha256=archive.archive_sha256, + ) + return MutationReport( + status="deleted" if changed_count else "already_absent", + event_count=len(archive.events), + changed_count=changed_count, + archive_sha256=archive.archive_sha256, + ) diff --git a/backend/app/governance/audit_retention_errors.py b/backend/app/governance/audit_retention_errors.py new file mode 100644 index 0000000..b2e5b5f --- /dev/null +++ b/backend/app/governance/audit_retention_errors.py @@ -0,0 +1,35 @@ +"""Dependency-free fixed outcome classes for the audit-retention CLI.""" + +from __future__ import annotations + + +class AuditRetentionError(RuntimeError): + """A fixed-code retention preflight or pre-commit failure.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class AuditRetentionOutcomeError(RuntimeError): + """Base class for outcomes that may already include a durable commit.""" + + def __init__( + self, + code: str, + *, + event_count: int | None = None, + archive_sha256: str | None = None, + ) -> None: + self.code = code + self.event_count = event_count + self.archive_sha256 = archive_sha256 + super().__init__(code) + + +class AuditRetentionCommitOutcomeUnknownError(AuditRetentionOutcomeError): + """The database did not confirm COMMIT after a successful flush.""" + + +class AuditRetentionCommittedVerificationError(AuditRetentionOutcomeError): + """COMMIT returned, but independent post-commit verification failed.""" diff --git a/backend/app/main.py b/backend/app/main.py index 71bbdce..3ac2b33 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -14,7 +14,14 @@ from app.api.v1.router import api_router from app.core.config import get_settings from app.core.constants import API_V1_PREFIX -from app.core.logging import REQUEST_ID_HEADER, configure_logging, new_correlation_id, request_scope +from app.core.logging import ( + REQUEST_ID_HEADER, + configure_logging, + new_correlation_id, + normalize_request_method, + normalize_request_path, + request_scope, +) from app.db.init_db import initialize_database from app.task_queue import create_run_queue @@ -143,8 +150,8 @@ async def correlate_request(request: Request, call_next): extra={ "event": "api_request_failed", "request_id": request_id, - "request_method": request.method, - "request_path": route_template, + "request_method": normalize_request_method(request.method), + "request_path": normalize_request_path(route_template), "duration_ms": round((monotonic() - started) * 1000, 3), "error_code": f"api_error:{type(exc).__name__}", "result": "internal_server_error", @@ -168,8 +175,8 @@ async def correlate_request(request: Request, call_next): extra={ "event": "api_request_completed", "request_id": request_id, - "request_method": request.method, - "request_path": route_template, + "request_method": normalize_request_method(request.method), + "request_path": normalize_request_path(route_template), "status_code": response.status_code, "duration_ms": round((monotonic() - started) * 1000, 3), }, diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index ecdcf0d..59446ba 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -25,6 +25,7 @@ from app.models.model import Model from app.models.model_credential import ModelCredential from app.models.question import Question +from app.models.worker_process import WorkerProcess __all__ = [ "TERMINAL_RUN_STATUSES", @@ -48,4 +49,5 @@ "QuestionExecution", "QuestionType", "RunStatus", + "WorkerProcess", ] diff --git a/backend/app/models/governance.py b/backend/app/models/governance.py index 950c54f..c112071 100644 --- a/backend/app/models/governance.py +++ b/backend/app/models/governance.py @@ -394,9 +394,11 @@ class AuditEvent(Base): CheckConstraint("lease_token IS NULL OR lease_token >= 0", name="lease_token_nonnegative"), CheckConstraint("duration_ms IS NULL OR duration_ms >= 0", name="duration_nonnegative"), CheckConstraint("expires_at > occurred_at", name="expiry_after_occurrence"), + Index("ix_audit_events_occurred_id", "occurred_at", "id"), Index("ix_audit_events_run_occurred", "run_id", "occurred_at", "id"), Index("ix_audit_events_type_occurred", "event_type", "occurred_at"), Index("ix_audit_events_expiry", "expires_at", "retention_class"), + Index("ix_audit_events_expires_id", "expires_at", "id"), ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=_uuid) diff --git a/backend/app/models/worker_process.py b/backend/app/models/worker_process.py new file mode 100644 index 0000000..6e42f57 --- /dev/null +++ b/backend/app/models/worker_process.py @@ -0,0 +1,57 @@ +"""Database-time facts for one long-running Worker process generation.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import CheckConstraint, Index, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base +from app.db.types import UTCDateTime + + +class WorkerProcess(Base): + """Low-volume lifecycle and coalesced main-loop progress facts.""" + + __tablename__ = "worker_processes" + __table_args__ = ( + UniqueConstraint("worker_id", name="uq_worker_processes_worker_id"), + CheckConstraint( + "length(worker_id) >= 1 AND length(worker_id) <= 128", + name="worker_id_length", + ), + CheckConstraint("last_seen_at >= started_at", name="seen_after_start"), + *( + CheckConstraint( + f"{column} IS NULL OR ({column} >= started_at AND {column} <= last_seen_at)", + name=f"{column}_within_seen", + ) + for column in ( + "last_scan_at", + "last_claim_at", + "last_progress_at", + "last_lease_heartbeat_at", + ) + ), + CheckConstraint( + "stopped_at IS NULL OR stopped_at >= last_seen_at", + name="stopped_after_seen", + ), + Index( + "ix_worker_processes_stopped_seen_generation", + "stopped_at", + "last_seen_at", + "generation_id", + ), + ) + + generation_id: Mapped[str] = mapped_column(String(36), primary_key=True) + worker_id: Mapped[str] = mapped_column(String(128), nullable=False) + started_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False) + last_scan_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + last_claim_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + last_progress_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + last_lease_heartbeat_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) + stopped_at: Mapped[datetime | None] = mapped_column(UTCDateTime()) diff --git a/backend/app/observability/__init__.py b/backend/app/observability/__init__.py new file mode 100644 index 0000000..8ca8f77 --- /dev/null +++ b/backend/app/observability/__init__.py @@ -0,0 +1,37 @@ +"""Database-derived, low-cardinality operational observations.""" + +from .snapshot import ( + METRICS_AUDIT_EVENT_LIMIT, + METRICS_AUDIT_WINDOW_SECONDS, + METRICS_LATENCY_SAMPLE_LIMIT, + METRICS_LATENCY_WINDOW_SECONDS, + TASK_EVENT_TYPES, + MetricsObservationLimitExceeded, + OperationalSnapshot, + TaskCurrentSnapshot, + TaskHistorySnapshot, + collect_operational_snapshot, + collect_task_current, + collect_task_history, + configure_read_snapshot, + database_clock, + latency_summary, +) + +__all__ = [ + "METRICS_AUDIT_EVENT_LIMIT", + "METRICS_AUDIT_WINDOW_SECONDS", + "METRICS_LATENCY_SAMPLE_LIMIT", + "METRICS_LATENCY_WINDOW_SECONDS", + "TASK_EVENT_TYPES", + "MetricsObservationLimitExceeded", + "OperationalSnapshot", + "TaskCurrentSnapshot", + "TaskHistorySnapshot", + "collect_operational_snapshot", + "collect_task_current", + "collect_task_history", + "configure_read_snapshot", + "database_clock", + "latency_summary", +] diff --git a/backend/app/observability/prometheus.py b/backend/app/observability/prometheus.py new file mode 100644 index 0000000..230902b --- /dev/null +++ b/backend/app/observability/prometheus.py @@ -0,0 +1,362 @@ +"""Strict Prometheus text exposition for the fixed Phase 2 metric set.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Iterable +from datetime import datetime + +from app.observability.snapshot import ( + METRICS_AUDIT_EVENT_LIMIT, + METRICS_AUDIT_WINDOW_SECONDS, + METRICS_LATENCY_SAMPLE_LIMIT, + METRICS_LATENCY_WINDOW_SECONDS, + TASK_EVENT_TYPES, + LatencySnapshot, + OperationalSnapshot, + as_utc, +) + +PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8" + + +class PrometheusRenderingError(RuntimeError): + """Raised when a snapshot cannot be represented without invalid samples.""" + + +_METRIC_NAME = re.compile(r"[a-zA-Z_:][a-zA-Z0-9_:]*") +_LABEL_NAME = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") +_FIXED_LABEL_VALUES = { + "event_type": frozenset(TASK_EVENT_TYPES), + "phase": frozenset({"queue", "execution", "end_to_end"}), + "quantile": frozenset({"0.5", "0.95", "0.99"}), + "state": frozenset({"registered", "live", "stalled"}), + "activity": frozenset({"scan", "claim", "lease_heartbeat", "progress"}), +} + + +def _number(value: object) -> str: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PrometheusRenderingError("metrics_value_type_invalid") + if isinstance(value, int): + if value < 0: + raise PrometheusRenderingError("metrics_value_not_finite_nonnegative") + return str(value) + numeric = float(value) + if not math.isfinite(numeric) or numeric < 0: + raise PrometheusRenderingError("metrics_value_not_finite_nonnegative") + if numeric == 0: + return "0" + return format(numeric, ".15g") + + +def _labels(values: tuple[tuple[str, str], ...]) -> str: + if not values: + return "" + rendered: list[str] = [] + for name, value in values: + allowed = _FIXED_LABEL_VALUES.get(name) + if ( + _LABEL_NAME.fullmatch(name) is None + or allowed is None + or value not in allowed + or any(character in value for character in ('"', "\\", "\n", "\r")) + ): + raise PrometheusRenderingError("metrics_label_invalid") + rendered.append(f'{name}="{value}"') + return "{" + ",".join(rendered) + "}" + + +def _family( + lines: list[str], + *, + name: str, + help_text: str, + samples: Iterable[tuple[tuple[tuple[str, str], ...], object]], +) -> None: + if _METRIC_NAME.fullmatch(name) is None or "\n" in help_text or "\r" in help_text: + raise PrometheusRenderingError("metrics_metadata_invalid") + lines.append(f"# HELP {name} {help_text}") + lines.append(f"# TYPE {name} gauge") + for labels, value in samples: + lines.append(f"{name}{_labels(labels)} {_number(value)}") + + +def _scalar(lines: list[str], name: str, help_text: str, value: object) -> None: + _family(lines, name=name, help_text=help_text, samples=(((), value),)) + + +def _latency_value(snapshot: LatencySnapshot, quantile: str) -> float | None: + if quantile == "0.5": + value = snapshot.p50_ms + elif quantile == "0.95": + value = snapshot.p95_ms + else: + value = snapshot.p99_ms + return None if value is None else value / 1000 + + +def _activity_age(now: datetime, observed_at: datetime | None) -> float | None: + if observed_at is None: + return None + age = (as_utc(now) - as_utc(observed_at)).total_seconds() + if age < 0: + raise PrometheusRenderingError("metrics_worker_activity_in_future") + return age + + +def render_prometheus( + snapshot: OperationalSnapshot, + *, + queue_configured: bool, + queue_available: bool, + recovery_alert_seconds: float, +) -> str: + """Render one fixed-order, low-cardinality exposition with no dynamic labels.""" + + current = snapshot.current + history = snapshot.history + lines: list[str] = [] + + current_metrics = ( + ("llmbenchlab_runs_pending", "Pending evaluation Runs.", current.pending), + ( + "llmbenchlab_runs_due_pending", + "Pending evaluation Runs currently eligible for dispatch.", + current.due_pending, + ), + ("llmbenchlab_runs_running", "Running evaluation Runs.", current.running), + ( + "llmbenchlab_runs_expired_running", + "Running evaluation Runs whose database lease has expired.", + current.expired_running, + ), + ( + "llmbenchlab_runs_cancellation_requested", + "Active Runs with cancellation requested.", + current.active_cancellation_requests, + ), + ( + "llmbenchlab_runs_retry_scheduled", + "Pending Runs with a database retry timestamp.", + current.retry_scheduled, + ), + ( + "llmbenchlab_runs_dead_lettered", + "Runs currently carrying a dead-letter timestamp.", + current.dead_lettered, + ), + ( + "llmbenchlab_runs_queue_notification_error", + "Runs carrying the fixed queue-notification-unavailable error.", + current.runs_with_queue_notification_error, + ), + ( + "llmbenchlab_runs_managed_backlog", + "Active managed Runs in the database backlog.", + current.managed_backlog, + ), + ( + "llmbenchlab_runs_governance_delayed", + "Runs currently delayed by governance.", + current.governance_delayed, + ), + ( + "llmbenchlab_runs_governance_exhausted", + "Runs currently exhausted by governance.", + current.governance_exhausted, + ), + ( + "llmbenchlab_provider_attempts_active", + "Provider attempt reservations not yet settled or released.", + current.active_provider_attempts, + ), + ( + "llmbenchlab_governance_scopes_overdrawn", + "Governance scopes marked overdrawn.", + current.overdrawn_governance_scopes, + ), + ( + "llmbenchlab_run_lease_acquisitions", + "Persisted Run lease acquisition count projection.", + current.total_attempts, + ), + ( + "llmbenchlab_run_failed_attempts", + "Persisted failed Run attempt count projection.", + current.total_failed_attempts, + ), + ( + "llmbenchlab_run_dispatches", + "Persisted Run dispatch count projection.", + current.total_dispatches, + ), + ( + "llmbenchlab_run_expired_lease_oldest_age_seconds", + "Right-censored age of the oldest currently expired running lease.", + current.expired_lease_oldest_age_seconds, + ), + ) + for name, help_text, value in current_metrics: + _scalar(lines, name, help_text, value) + + _family( + lines, + name="llmbenchlab_audit_events_window", + help_text="Validated retained audit events in the fixed rolling window.", + samples=( + ((("event_type", event_type),), history.event_counts[event_type]) + for event_type in TASK_EVENT_TYPES + ), + ) + _scalar( + lines, + "llmbenchlab_audit_event_window_seconds", + "Fixed rolling audit-event observation window.", + METRICS_AUDIT_WINDOW_SECONDS, + ) + _scalar( + lines, + "llmbenchlab_metrics_audit_events_scanned", + "Retained audit rows validated during this collection.", + history.audit_events_scanned, + ) + _scalar( + lines, + "llmbenchlab_metrics_audit_event_limit", + "Hard retained audit row limit for one collection.", + METRICS_AUDIT_EVENT_LIMIT, + ) + + latency_snapshots = ( + ("queue", history.queue_latency), + ("execution", history.execution_latency), + ("end_to_end", history.end_to_end_latency), + ) + quantile_samples = [] + for phase, latency in latency_snapshots: + for quantile in ("0.5", "0.95", "0.99"): + value = _latency_value(latency, quantile) + if value is not None: + quantile_samples.append(((("phase", phase), ("quantile", quantile)), value)) + _family( + lines, + name="llmbenchlab_run_latency_quantile_seconds", + help_text="Run latency quantile over the fixed latency window.", + samples=quantile_samples, + ) + _family( + lines, + name="llmbenchlab_run_latency_samples", + help_text="Run latency samples used after the stable sample cap.", + samples=( + ((("phase", phase),), latency.sample_count) for phase, latency in latency_snapshots + ), + ) + _family( + lines, + name="llmbenchlab_run_latency_truncated", + help_text="Whether the Run latency sample page was truncated.", + samples=( + ((("phase", phase),), int(latency.truncated)) for phase, latency in latency_snapshots + ), + ) + _scalar( + lines, + "llmbenchlab_run_latency_window_seconds", + "Fixed Run latency observation window.", + METRICS_LATENCY_WINDOW_SECONDS, + ) + _scalar( + lines, + "llmbenchlab_metrics_latency_sample_limit", + "Hard sample limit for each Run latency phase.", + METRICS_LATENCY_SAMPLE_LIMIT, + ) + + _scalar( + lines, + "llmbenchlab_queue_configured", + "Whether a Redis notification queue is configured.", + int(queue_configured), + ) + _scalar( + lines, + "llmbenchlab_queue_available", + "Whether the configured Redis notification queue answered this scrape.", + int(queue_available), + ) + worker = current.worker + _family( + lines, + name="llmbenchlab_worker_processes", + help_text="Registered Worker generations by derived database-time state.", + samples=( + ((("state", "registered"),), worker.registered), + ((("state", "live"),), worker.live), + ((("state", "stalled"),), worker.stalled), + ), + ) + _scalar( + lines, + "llmbenchlab_worker_expected_minimum", + "Configured minimum number of live Worker processes.", + worker.expected, + ) + _scalar( + lines, + "llmbenchlab_worker_shortfall", + "Configured live Worker process shortfall.", + worker.shortfall, + ) + oldest_activity = current.worker_activity_oldest + activities = ( + ("scan", oldest_activity.scan_at), + ("claim", oldest_activity.claim_at), + ("lease_heartbeat", oldest_activity.lease_heartbeat_at), + ("progress", oldest_activity.progress_at), + ) + activity_ages = tuple( + (activity, _activity_age(current.timestamp, observed_at)) + for activity, observed_at in activities + ) + _family( + lines, + name="llmbenchlab_worker_activity_observed", + help_text="Whether an active Worker has recorded this activity.", + samples=( + ((("activity", activity),), int(age is not None)) for activity, age in activity_ages + ), + ) + _family( + lines, + name="llmbenchlab_worker_activity_oldest_age_seconds", + help_text="Database-time age of the oldest active-generation Worker activity fact.", + samples=( + ((("activity", activity),), age) for activity, age in activity_ages if age is not None + ), + ) + _scalar( + lines, + "llmbenchlab_worker_stale_threshold_seconds", + "Configured database-time Worker stale threshold.", + worker.stale_seconds, + ) + _scalar( + lines, + "llmbenchlab_run_recovery_alert_threshold_seconds", + "Configured alert threshold for an expired running lease.", + recovery_alert_seconds, + ) + _scalar( + lines, + "llmbenchlab_metrics_snapshot_unixtime_seconds", + "Database snapshot timestamp as Unix seconds.", + as_utc(current.timestamp).timestamp(), + ) + + rendered = "\n".join(lines) + "\n" + if "\r" in rendered or not rendered.endswith("\n") or rendered.endswith("\n\n"): + raise PrometheusRenderingError("metrics_text_framing_invalid") + return rendered diff --git a/backend/app/observability/snapshot.py b/backend/app/observability/snapshot.py new file mode 100644 index 0000000..dde9d67 --- /dev/null +++ b/backend/app/observability/snapshot.py @@ -0,0 +1,542 @@ +"""Bounded database snapshots shared by JSON task APIs and Prometheus.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import case, func, select +from sqlalchemy.orm import Session + +from app.governance import AuditIntegrityError, validate_audit_event_for_read +from app.models import ( + AuditEvent, + EvaluationRun, + GovernanceRunStatus, + GovernanceScope, + ProviderCallReservation, + ProviderCallReservationState, + RunStatus, + WorkerProcess, +) +from app.worker_progress import WorkerProgressSnapshot, collect_worker_progress + +TASK_EVENT_TYPES = ( + "governance_policy_bootstrapped", + "governance_policy_applied", + "run_admitted", + "run_claimed", + "run_cancel_requested", + "run_deferred", + "run_yielded", + "run_terminal", + "run_retry_scheduled", + "run_dead_lettered", + "run_lease_reconciled", + "provider_attempt_reserved", + "provider_attempt_send_started", + "provider_attempt_settled", + "question_evidence_persisted", + "queue_notification", + "governance_integrity_error", +) + +METRICS_AUDIT_WINDOW_SECONDS = 15 * 60 +METRICS_AUDIT_EVENT_LIMIT = 50_000 +METRICS_LATENCY_WINDOW_SECONDS = 60 * 60 +METRICS_LATENCY_SAMPLE_LIMIT = 10_000 + + +class MetricsObservationLimitExceeded(RuntimeError): + """Raised instead of returning a truncated rolling audit count.""" + + +@dataclass(frozen=True, slots=True) +class LatencySnapshot: + """One bounded Run latency distribution in milliseconds.""" + + sample_count: int + truncated: bool + p50_ms: float | None + p95_ms: float | None + p99_ms: float | None + + +@dataclass(frozen=True, slots=True) +class WorkerActivityOldestSnapshot: + """Oldest activity timestamp across currently registered generations.""" + + scan_at: datetime | None + claim_at: datetime | None + lease_heartbeat_at: datetime | None + progress_at: datetime | None + + +@dataclass(frozen=True, slots=True) +class TaskCurrentSnapshot: + """Current database gauges at one caller-supplied database timestamp.""" + + pending: int + due_pending: int + running: int + expired_running: int + active_cancellation_requests: int + retry_scheduled: int + dead_lettered: int + runs_with_queue_notification_error: int + managed_backlog: int + governance_delayed: int + governance_exhausted: int + active_provider_attempts: int + overdrawn_governance_scopes: int + total_attempts: int + total_failed_attempts: int + total_dispatches: int + expired_lease_oldest_age_seconds: float + worker: WorkerProgressSnapshot + worker_activity_oldest: WorkerActivityOldestSnapshot + timestamp: datetime + + +@dataclass(frozen=True, slots=True) +class TaskHistorySnapshot: + """Validated typed-event counts and three bounded Run distributions.""" + + window_start: datetime + window_end: datetime + event_counts: dict[str, int] + audit_events_scanned: int + queue_latency: LatencySnapshot + execution_latency: LatencySnapshot + end_to_end_latency: LatencySnapshot + latency_sample_limit: int + + +@dataclass(frozen=True, slots=True) +class OperationalSnapshot: + """Fixed exporter read model collected within one database transaction.""" + + current: TaskCurrentSnapshot + history: TaskHistorySnapshot + + +def as_utc(value: datetime) -> datetime: + """Normalize a driver timestamp without consulting host-local time.""" + + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def database_clock(session: Session) -> datetime: + """Read the authoritative database clock.""" + + value = session.scalar(select(func.current_timestamp())) + if not isinstance(value, datetime): + raise RuntimeError("database_timestamp_unavailable") + return as_utc(value) + + +def configure_read_snapshot(session: Session) -> None: + """Start the strongest portable read snapshot before the first query.""" + + dialect = session.get_bind().dialect.name + connection = session.connection() + if dialect == "sqlite": + # SQLite's legacy driver mode does not begin a transaction for SELECT. + connection.exec_driver_sql("BEGIN") + elif dialect == "postgresql": + connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + + +def percentile(values: Sequence[float], percentage: float) -> float: + """Return a linearly interpolated percentile over finite caller data.""" + + ordered = sorted(float(value) for value in values) + if not ordered: + raise ValueError("percentile_requires_sample") + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * percentage / 100 + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +def latency_summary( + values: Sequence[float], + *, + sample_limit: int = METRICS_LATENCY_SAMPLE_LIMIT, +) -> LatencySnapshot: + """Summarize the first stable sample page and disclose truncation.""" + + if sample_limit < 1: + raise ValueError("latency_sample_limit_must_be_positive") + truncated = len(values) > sample_limit + selected = list(values[:sample_limit]) + if not selected: + return LatencySnapshot( + sample_count=0, + truncated=False, + p50_ms=None, + p95_ms=None, + p99_ms=None, + ) + return LatencySnapshot( + sample_count=len(selected), + truncated=truncated, + p50_ms=round(percentile(selected, 50), 6), + p95_ms=round(percentile(selected, 95), 6), + p99_ms=round(percentile(selected, 99), 6), + ) + + +def _run_latency_summary( + session: Session, + *, + observed_at: Any, + started_at: Any, + ended_at: Any, + window_start: datetime, + window_end: datetime, + sample_limit: int, +) -> LatencySnapshot: + rows = session.execute( + select(started_at, ended_at) + .where( + observed_at >= window_start, + observed_at < window_end, + started_at.is_not(None), + ended_at.is_not(None), + ended_at >= started_at, + ) + .order_by(observed_at, EvaluationRun.id) + .limit(sample_limit + 1) + ).all() + values = [ + (as_utc(end_value) - as_utc(start_value)).total_seconds() * 1000 + for start_value, end_value in rows + ] + return latency_summary(values, sample_limit=sample_limit) + + +def collect_task_current( + session: Session, + *, + now: datetime, + worker_stale_seconds: float, + worker_expected_processes: int, +) -> TaskCurrentSnapshot: + """Collect current Run, governance, lease, and Worker aggregates.""" + + now = as_utc(now) + active_run_statuses = (RunStatus.PENDING, RunStatus.RUNNING) + row = session.execute( + select( + func.coalesce( + func.sum(case((EvaluationRun.status == RunStatus.PENDING, 1), else_=0)), 0 + ).label("pending"), + func.coalesce( + func.sum( + case( + ( + (EvaluationRun.status == RunStatus.PENDING) + & ( + EvaluationRun.next_attempt_at.is_(None) + | (EvaluationRun.next_attempt_at <= now) + ) + & ( + EvaluationRun.governance_not_before.is_(None) + | (EvaluationRun.governance_not_before <= now) + ), + 1, + ), + else_=0, + ) + ), + 0, + ).label("due_pending"), + func.coalesce( + func.sum(case((EvaluationRun.status == RunStatus.RUNNING, 1), else_=0)), 0 + ).label("running"), + func.coalesce( + func.sum( + case( + ( + (EvaluationRun.status == RunStatus.RUNNING) + & (EvaluationRun.lease_expires_at <= now), + 1, + ), + else_=0, + ) + ), + 0, + ).label("expired_running"), + func.coalesce( + func.sum( + case( + ( + EvaluationRun.cancellation_requested.is_(True) + & EvaluationRun.status.in_(active_run_statuses), + 1, + ), + else_=0, + ) + ), + 0, + ).label("active_cancellation_requests"), + func.coalesce( + func.sum( + case( + ( + (EvaluationRun.status == RunStatus.PENDING) + & EvaluationRun.next_attempt_at.is_not(None), + 1, + ), + else_=0, + ) + ), + 0, + ).label("retry_scheduled"), + func.coalesce( + func.sum(case((EvaluationRun.dead_lettered_at.is_not(None), 1), else_=0)), 0 + ).label("dead_lettered"), + func.coalesce( + func.sum( + case((EvaluationRun.last_error == "queue_notification_unavailable", 1), else_=0) + ), + 0, + ).label("runs_with_queue_notification_error"), + func.coalesce( + func.sum( + case( + ( + EvaluationRun.status.in_(active_run_statuses) + & ( + EvaluationRun.governance_status + != GovernanceRunStatus.LEGACY_UNMANAGED + ), + 1, + ), + else_=0, + ) + ), + 0, + ).label("managed_backlog"), + func.coalesce( + func.sum( + case( + (EvaluationRun.governance_status == GovernanceRunStatus.DELAYED, 1), + else_=0, + ) + ), + 0, + ).label("governance_delayed"), + func.coalesce( + func.sum( + case( + (EvaluationRun.governance_status == GovernanceRunStatus.EXHAUSTED, 1), + else_=0, + ) + ), + 0, + ).label("governance_exhausted"), + func.coalesce(func.sum(EvaluationRun.attempt_count), 0).label("total_attempts"), + func.coalesce(func.sum(EvaluationRun.failed_attempt_count), 0).label( + "total_failed_attempts" + ), + func.coalesce(func.sum(EvaluationRun.dispatch_count), 0).label("total_dispatches"), + func.min( + case( + ( + (EvaluationRun.status == RunStatus.RUNNING) + & (EvaluationRun.lease_expires_at <= now), + EvaluationRun.lease_expires_at, + ), + else_=None, + ) + ).label("oldest_expired_lease_at"), + ) + ).one() + active_provider_attempts = int( + session.scalar( + select(func.count(ProviderCallReservation.id)).where( + ProviderCallReservation.state.in_( + ( + ProviderCallReservationState.RESERVED, + ProviderCallReservationState.SEND_STARTED, + ) + ) + ) + ) + or 0 + ) + overdrawn_governance_scopes = int( + session.scalar( + select(func.count(GovernanceScope.id)).where(GovernanceScope.overdrawn.is_(True)) + ) + or 0 + ) + worker = collect_worker_progress( + session, + now=now, + stale_seconds=worker_stale_seconds, + expected=worker_expected_processes, + ) + active_worker = WorkerProcess.stopped_at.is_(None) + worker_activity_row = session.execute( + select( + func.min(case((active_worker, WorkerProcess.last_scan_at), else_=None)), + func.min(case((active_worker, WorkerProcess.last_claim_at), else_=None)), + func.min(case((active_worker, WorkerProcess.last_lease_heartbeat_at), else_=None)), + func.min(case((active_worker, WorkerProcess.last_progress_at), else_=None)), + ) + ).one() + worker_activity_oldest = WorkerActivityOldestSnapshot( + scan_at=as_utc(worker_activity_row[0]) if worker_activity_row[0] is not None else None, + claim_at=as_utc(worker_activity_row[1]) if worker_activity_row[1] is not None else None, + lease_heartbeat_at=( + as_utc(worker_activity_row[2]) if worker_activity_row[2] is not None else None + ), + progress_at=( + as_utc(worker_activity_row[3]) if worker_activity_row[3] is not None else None + ), + ) + oldest_expired = row.oldest_expired_lease_at + expired_age = ( + max(0.0, (now - as_utc(oldest_expired)).total_seconds()) + if isinstance(oldest_expired, datetime) + else 0.0 + ) + return TaskCurrentSnapshot( + pending=int(row.pending), + due_pending=int(row.due_pending), + running=int(row.running), + expired_running=int(row.expired_running), + active_cancellation_requests=int(row.active_cancellation_requests), + retry_scheduled=int(row.retry_scheduled), + dead_lettered=int(row.dead_lettered), + runs_with_queue_notification_error=int(row.runs_with_queue_notification_error), + managed_backlog=int(row.managed_backlog), + governance_delayed=int(row.governance_delayed), + governance_exhausted=int(row.governance_exhausted), + active_provider_attempts=active_provider_attempts, + overdrawn_governance_scopes=overdrawn_governance_scopes, + total_attempts=int(row.total_attempts), + total_failed_attempts=int(row.total_failed_attempts), + total_dispatches=int(row.total_dispatches), + expired_lease_oldest_age_seconds=expired_age, + worker=worker, + worker_activity_oldest=worker_activity_oldest, + timestamp=now, + ) + + +def collect_task_history( + session: Session, + *, + window_start: datetime, + window_end: datetime, + audit_event_limit: int | None, + latency_sample_limit: int = METRICS_LATENCY_SAMPLE_LIMIT, + latency_window_start: datetime | None = None, +) -> TaskHistorySnapshot: + """Validate and aggregate one half-open UTC window from retained facts.""" + + if audit_event_limit is not None and audit_event_limit < 1: + raise ValueError("audit_event_limit_must_be_positive") + if latency_sample_limit < 1: + raise ValueError("latency_sample_limit_must_be_positive") + effective_latency_start = latency_window_start or window_start + event_query = ( + select(AuditEvent) + .where( + AuditEvent.occurred_at >= window_start, + AuditEvent.occurred_at < window_end, + ) + .order_by(AuditEvent.occurred_at, AuditEvent.id) + ) + if audit_event_limit is not None: + event_query = event_query.limit(audit_event_limit + 1) + try: + events = list(session.scalars(event_query)) + except (LookupError, OverflowError, TypeError, ValueError): + # SQLite can contain invalid typed values after an out-of-band write. + # ORM JSON/enum/timestamp deserialization happens before the retained-row + # validator sees the event, so preserve the same integrity classification + # at this narrowly scoped AuditEvent hydration boundary. + raise AuditIntegrityError("retained audit event failed integrity validation") from None + if audit_event_limit is not None and len(events) > audit_event_limit: + raise MetricsObservationLimitExceeded("metrics_audit_event_limit_exceeded") + + event_values = {event_type: 0 for event_type in TASK_EVENT_TYPES} + for event in events: + facts = validate_audit_event_for_read(event) + if facts.event_type in event_values: + event_values[facts.event_type] += 1 + + return TaskHistorySnapshot( + window_start=window_start, + window_end=window_end, + event_counts=event_values, + audit_events_scanned=len(events), + queue_latency=_run_latency_summary( + session, + observed_at=EvaluationRun.started_at, + started_at=EvaluationRun.created_at, + ended_at=EvaluationRun.started_at, + window_start=effective_latency_start, + window_end=window_end, + sample_limit=latency_sample_limit, + ), + execution_latency=_run_latency_summary( + session, + observed_at=EvaluationRun.finished_at, + started_at=EvaluationRun.started_at, + ended_at=EvaluationRun.finished_at, + window_start=effective_latency_start, + window_end=window_end, + sample_limit=latency_sample_limit, + ), + end_to_end_latency=_run_latency_summary( + session, + observed_at=EvaluationRun.finished_at, + started_at=EvaluationRun.created_at, + ended_at=EvaluationRun.finished_at, + window_start=effective_latency_start, + window_end=window_end, + sample_limit=latency_sample_limit, + ), + latency_sample_limit=latency_sample_limit, + ) + + +def collect_operational_snapshot( + session: Session, + *, + worker_stale_seconds: float, + worker_expected_processes: int, +) -> OperationalSnapshot: + """Collect the fixed exporter snapshot using one transaction and DB clock.""" + + with session.begin(): + configure_read_snapshot(session) + now = database_clock(session) + current = collect_task_current( + session, + now=now, + worker_stale_seconds=worker_stale_seconds, + worker_expected_processes=worker_expected_processes, + ) + history = collect_task_history( + session, + window_start=now - timedelta(seconds=METRICS_AUDIT_WINDOW_SECONDS), + window_end=now, + audit_event_limit=METRICS_AUDIT_EVENT_LIMIT, + latency_sample_limit=METRICS_LATENCY_SAMPLE_LIMIT, + latency_window_start=now - timedelta(seconds=METRICS_LATENCY_WINDOW_SECONDS), + ) + return OperationalSnapshot(current=current, history=history) diff --git a/backend/app/runners/evaluation_runner.py b/backend/app/runners/evaluation_runner.py index fcbe754..0a33105 100644 --- a/backend/app/runners/evaluation_runner.py +++ b/backend/app/runners/evaluation_runner.py @@ -56,6 +56,11 @@ normalize_provider_metadata, ) from app.services.credential_audit import audit_credential_decrypt_failed +from app.worker_progress import ( + NULL_WORKER_PROGRESS_OBSERVER, + WorkerProgressEvent, + WorkerProgressObserver, +) logger = logging.getLogger(__name__) @@ -118,6 +123,7 @@ def __init__( worker_id: str | None = None, lease_repository: RunLeaseRepository | None = None, governance_repository: GovernanceRepository | None = None, + progress_observer: WorkerProgressObserver | None = None, ) -> None: settings = get_settings() self._settings = settings @@ -132,6 +138,7 @@ def __init__( self._governance_repository = governance_repository or GovernanceRepository(session_factory) self._heartbeat_seconds = settings.worker_heartbeat_seconds self._mock_generation_delay_seconds = settings.mock_generation_delay_seconds + self._progress_observer = progress_observer or NULL_WORKER_PROGRESS_OBSERVER async def execute( self, @@ -166,6 +173,7 @@ async def execute( "result": "claimed", }, ) + self._note_progress(WorkerProgressEvent.CLAIM) heartbeat_stop = asyncio.Event() lease_lost = asyncio.Event() heartbeat_task = asyncio.create_task( @@ -234,6 +242,7 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: ) if response_disposition == ResponseDisposition.INSERTED: responses_added += 1 + self._note_progress(WorkerProgressEvent.PROGRESS) return True governance_signal = await self._run_questions( @@ -290,6 +299,7 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: ) if disposition == AttemptDisposition.FENCE_LOST: raise _LeaseUnavailable("governance_transition_fence_lost") + self._note_progress(WorkerProgressEvent.PROGRESS) return True if not cancellation_requested and model.governance is not None and questions_remain: disposition = await asyncio.to_thread( @@ -310,9 +320,12 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: ) if disposition == AttemptDisposition.FENCE_LOST: raise _LeaseUnavailable("cooperative_yield_fence_lost") + self._note_progress(WorkerProgressEvent.PROGRESS) return True final_status = RunStatus.CANCELLED if cancellation_requested else RunStatus.COMPLETED finished_status = await asyncio.to_thread(self._finish, lease, final_status) + if finished_status is not None: + self._note_progress(WorkerProgressEvent.PROGRESS) logger.info( "Evaluation run finish transition resolved", extra={ @@ -353,6 +366,7 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: return False except _LeaseUnavailable: if await asyncio.to_thread(self._lease_repository.finish_cancelled, lease): + self._note_progress(WorkerProgressEvent.PROGRESS) logger.info( "Evaluation run stopped after cancellation", extra={ @@ -395,6 +409,8 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: lease, error_code=f"runner_error:{type(exc).__name__}", ) + if disposition != AttemptDisposition.FENCE_LOST: + self._note_progress(WorkerProgressEvent.PROGRESS) logger.warning( "Evaluation run failure transition resolved", extra={ @@ -434,6 +450,22 @@ async def evaluate_bounded(question: _QuestionSnapshot) -> bool: def _claim(self, run_id: str) -> RunLease | None: return self._lease_repository.claim(run_id, owner=self._worker_id) + def _note_progress(self, event: WorkerProgressEvent) -> None: + """Keep optional observability from changing the execution state machine.""" + + try: + observer = getattr(self, "_progress_observer", NULL_WORKER_PROGRESS_OBSERVER) + observer.note(event) + except Exception as exc: + logger.warning( + "Worker progress observer rejected an execution fact", + extra={ + "event": "worker_progress_observer_failed", + "error_code": f"worker_progress_error:{type(exc).__name__}", + "result": "ignored", + }, + ) + async def _heartbeat( self, lease: RunLease, @@ -485,6 +517,7 @@ async def _heartbeat( "result": "renewed", }, ) + self._note_progress(WorkerProgressEvent.LEASE_HEARTBEAT) @staticmethod async def _run_questions( diff --git a/backend/app/schemas/system.py b/backend/app/schemas/system.py index 456d10a..412183e 100644 --- a/backend/app/schemas/system.py +++ b/backend/app/schemas/system.py @@ -53,6 +53,17 @@ class TaskMetricsResponse(ORMModel): total_attempts: int total_failed_attempts: int total_dispatches: int + worker_expected_processes: int + worker_registered_processes: int + worker_live_processes: int + worker_stalled_processes: int + worker_shortfall_processes: int + worker_stale_after_seconds: float + worker_last_seen_at: datetime | None + worker_last_scan_at: datetime | None + worker_last_claim_at: datetime | None + worker_last_progress_at: datetime | None + worker_last_lease_heartbeat_at: datetime | None timestamp: datetime diff --git a/backend/app/task_queue/redis_streams.py b/backend/app/task_queue/redis_streams.py index 225f6ef..a7c8a60 100644 --- a/backend/app/task_queue/redis_streams.py +++ b/backend/app/task_queue/redis_streams.py @@ -10,6 +10,12 @@ from redis.asyncio import Redis from redis.exceptions import RedisError, ResponseError +from app.core.logging import ( + normalize_correlation_id, + normalize_redis_stream_id, + normalize_uuid_identifier, +) + TASK_MESSAGE_VERSION = "llmbenchlab-run-task-v1" @@ -105,23 +111,28 @@ async def ensure_consumer_group(self) -> None: raise QueueUnavailable("queue_consumer_group_unavailable") from exc async def publish(self, run_id: str, *, correlation_id: str) -> str: - if not 1 <= len(run_id) <= 128 or not 1 <= len(correlation_id) <= 128: - raise ValueError("run_id and correlation_id must contain 1 to 128 characters") + normalized_run_id = normalize_uuid_identifier(run_id) + normalized_correlation_id = normalize_correlation_id(correlation_id) + if normalized_run_id is None or normalized_correlation_id is None: + raise ValueError("run_id and correlation_id must be canonical UUIDs") try: async with asyncio.timeout(self._publish_timeout_seconds): - message_id = await self._client.xadd( + raw_message_id = await self._client.xadd( self.stream, { "version": TASK_MESSAGE_VERSION, - "run_id": run_id, - "correlation_id": correlation_id, + "run_id": normalized_run_id, + "correlation_id": normalized_correlation_id, }, maxlen=self._max_length, approximate=True, ) except (RedisError, OSError, TimeoutError) as exc: raise QueueUnavailable("queue_publish_unavailable") from exc - return _text(message_id) + message_id = normalize_redis_stream_id(_text(raw_message_id)) + if message_id is None: + raise QueueUnavailable("queue_publish_unavailable") + return message_id async def read_new( self, @@ -207,14 +218,12 @@ def _first_delivery(self, response: Any) -> RunTaskDelivery | None: @staticmethod def _delivery(message_id: Any, fields: Mapping[Any, Any]) -> RunTaskDelivery: normalized = {_text(key): _text(value) for key, value in fields.items()} - run_id = normalized.get("run_id") - correlation_id = normalized.get("correlation_id") + run_id = normalize_uuid_identifier(normalized.get("run_id")) + correlation_id = normalize_correlation_id(normalized.get("correlation_id")) if ( normalized.get("version") != TASK_MESSAGE_VERSION or run_id is None - or not 1 <= len(run_id) <= 128 or correlation_id is None - or not 1 <= len(correlation_id) <= 128 ): run_id = None correlation_id = None diff --git a/backend/app/worker.py b/backend/app/worker.py index c162368..4f37c99 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -13,7 +13,10 @@ from app.task_queue import create_run_queue from app.workers import WorkerService -logger = logging.getLogger(__name__) +# This module is executed as ``python -m app.worker`` in production, where +# ``__name__`` is ``__main__``. Keep the registered application logger stable +# so the final formatter preserves the closed Worker event contract. +logger = logging.getLogger("app.worker") async def _run_worker() -> int: @@ -38,11 +41,24 @@ async def _run_worker() -> int: continue installed_signals.append(signal_name) logger.info( - "Worker started", - extra={"event": "worker_started", "worker_id": service.worker_id}, + "Worker process is starting", + extra={"event": "worker_starting", "worker_id": service.worker_id}, ) try: - await service.run(stop) + try: + await service.run(stop) + except Exception as exc: + logger.error( + "Worker main service stopped before a healthy lifecycle completed", + extra={ + "event": "worker_service_failed", + "error_code": f"worker_service_error:{type(exc).__name__}", + "result": "stopped", + }, + ) + return_code = 1 + else: + return_code = 0 finally: for signal_name in installed_signals: loop.remove_signal_handler(signal_name) @@ -54,7 +70,7 @@ async def _run_worker() -> int: "Worker stopped", extra={"event": "worker_stopped", "worker_id": service.worker_id}, ) - return 0 + return return_code def main() -> None: diff --git a/backend/app/worker_probe.py b/backend/app/worker_probe.py index 944787f..681f8f1 100644 --- a/backend/app/worker_probe.py +++ b/backend/app/worker_probe.py @@ -18,7 +18,16 @@ async def _probe() -> int: try: initialize_database() except Exception: - print(json.dumps({"status": "not_ready", "database": "unavailable"})) + print( + json.dumps( + { + "status": "not_ready", + "database": "unavailable", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", + } + ) + ) engine.dispose() return 1 @@ -33,6 +42,8 @@ async def _probe() -> int: "database": "ok", "schema": "ok", "queue": "configuration_error", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", } ) ) @@ -54,6 +65,8 @@ async def _probe() -> int: "schema": "ok", "queue": queue_status, "database_reconciliation": "available", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", } ) ) diff --git a/backend/app/worker_progress.py b/backend/app/worker_progress.py new file mode 100644 index 0000000..e59441d --- /dev/null +++ b/backend/app/worker_progress.py @@ -0,0 +1,354 @@ +"""Coalesced DB-time Worker main-loop progress and its aggregate read model.""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import IntFlag, StrEnum +from threading import Lock +from typing import Protocol +from uuid import UUID, uuid4 + +from sqlalchemy import case, func, select, update +from sqlalchemy.orm import Session, sessionmaker + +from app.db.clock import database_utc_now +from app.models import WorkerProcess + +logger = logging.getLogger(__name__) + + +class WorkerProgressEvent(StrEnum): + """Allowlisted durable activities accepted by a Worker progress observer.""" + + SCAN = "scan" + CLAIM = "claim" + PROGRESS = "progress" + LEASE_HEARTBEAT = "lease_heartbeat" + + +class WorkerProgressObserver(Protocol): + """No-throw note interface used at post-commit execution boundaries.""" + + def note(self, event: WorkerProgressEvent) -> None: ... + + +class WorkerProgressLifecycle(WorkerProgressObserver, Protocol): + """Lifecycle used only by the long-running Worker service.""" + + async def start(self) -> None: ... + + async def stop(self) -> bool: ... + + +class NullWorkerProgressObserver: + """Compatibility observer for runners and tools without a long-lived Worker.""" + + def note(self, event: WorkerProgressEvent) -> None: + del event + + +NULL_WORKER_PROGRESS_OBSERVER = NullWorkerProgressObserver() + + +class WorkerProgressStateError(RuntimeError): + """Raised when a generation is missing, stopped, or otherwise not writable.""" + + +class _PendingEvent(IntFlag): + NONE = 0 + SCAN = 1 + CLAIM = 2 + PROGRESS = 4 + LEASE_HEARTBEAT = 8 + + +_EVENT_BITS = { + WorkerProgressEvent.SCAN: _PendingEvent.SCAN, + WorkerProgressEvent.CLAIM: _PendingEvent.CLAIM, + WorkerProgressEvent.PROGRESS: _PendingEvent.PROGRESS, + WorkerProgressEvent.LEASE_HEARTBEAT: _PendingEvent.LEASE_HEARTBEAT, +} + + +@dataclass(frozen=True, slots=True) +class WorkerProgressSnapshot: + """Low-cardinality aggregate at one caller-supplied database timestamp.""" + + expected: int + registered: int + live: int + stalled: int + shortfall: int + stale_seconds: float + last_seen_at: datetime | None + last_scan_at: datetime | None + last_claim_at: datetime | None + last_progress_at: datetime | None + last_lease_heartbeat_at: datetime | None + + +def _as_utc(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def collect_worker_progress( + session: Session, + *, + now: datetime, + stale_seconds: float, + expected: int, +) -> WorkerProgressSnapshot: + """Aggregate active generations without exposing process or Worker identifiers.""" + + if stale_seconds <= 0 or expected < 0: + raise ValueError("Worker progress aggregate inputs must be non-negative") + normalized_now = now.replace(tzinfo=UTC) if now.tzinfo is None else now.astimezone(UTC) + cutoff = normalized_now - timedelta(seconds=stale_seconds) + active = WorkerProcess.stopped_at.is_(None) + row = session.execute( + select( + func.count(WorkerProcess.generation_id).filter(active), + func.count(WorkerProcess.generation_id).filter( + active, WorkerProcess.last_seen_at >= cutoff + ), + func.count(WorkerProcess.generation_id).filter( + active, WorkerProcess.last_seen_at < cutoff + ), + func.max(case((active, WorkerProcess.last_seen_at), else_=None)), + func.max(case((active, WorkerProcess.last_scan_at), else_=None)), + func.max(case((active, WorkerProcess.last_claim_at), else_=None)), + func.max(case((active, WorkerProcess.last_progress_at), else_=None)), + func.max(case((active, WorkerProcess.last_lease_heartbeat_at), else_=None)), + ) + ).one() + registered = int(row[0] or 0) + live = int(row[1] or 0) + stalled = int(row[2] or 0) + return WorkerProgressSnapshot( + expected=expected, + registered=registered, + live=live, + stalled=stalled, + shortfall=max(expected - live, 0), + stale_seconds=stale_seconds, + last_seen_at=_as_utc(row[3]), + last_scan_at=_as_utc(row[4]), + last_claim_at=_as_utc(row[5]), + last_progress_at=_as_utc(row[6]), + last_lease_heartbeat_at=_as_utc(row[7]), + ) + + +class WorkerProgressRecorder: + """Register one generation and coalesce real progress into short transactions.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + *, + worker_id: str, + flush_seconds: float, + generation_id: str | None = None, + ) -> None: + if not worker_id or len(worker_id) > 128: + raise ValueError("worker_id must contain 1 to 128 characters") + if flush_seconds <= 0: + raise ValueError("flush_seconds must be positive") + canonical_generation = generation_id or str(uuid4()) + try: + parsed_generation = UUID(canonical_generation) + except (TypeError, ValueError) as exc: + raise ValueError("generation_id must be a canonical UUID") from exc + if str(parsed_generation) != canonical_generation: + raise ValueError("generation_id must be a canonical UUID") + self._session_factory = session_factory + self._worker_id = worker_id + self.generation_id = canonical_generation + self._flush_seconds = flush_seconds + self._state_lock = Lock() + self._pending = _PendingEvent.NONE + self._registered = False + self._stopping = False + self._stopped = False + self._flush_stop: asyncio.Event | None = None + self._flush_task: asyncio.Task[None] | None = None + self._flush_lock: asyncio.Lock | None = None + + def note(self, event: WorkerProgressEvent) -> None: + """Record an allowlisted event in memory; this method deliberately never raises.""" + + bit = _EVENT_BITS.get(event) + if bit is None: + return + with self._state_lock: + if self._registered and not self._stopping and not self._stopped: + self._pending |= bit + + async def start(self) -> None: + """Register before execution and start the sole periodic flush loop.""" + + if self._registered or self._flush_task is not None: + raise WorkerProgressStateError("worker_progress_already_started") + await asyncio.to_thread(self._register) + with self._state_lock: + self._registered = True + self._flush_stop = asyncio.Event() + self._flush_lock = asyncio.Lock() + self._flush_task = asyncio.create_task(self._flush_loop(), name="worker-progress-flush") + + async def flush_now(self) -> bool: + """Try one coalesced flush; false means no event or a conservative retry.""" + + flush_lock = self._flush_lock + if flush_lock is None: + return False + async with flush_lock: + return await self._flush_once() + + async def _flush_once(self) -> bool: + with self._state_lock: + if ( + not self._registered + or self._stopping + or self._stopped + or self._pending == _PendingEvent.NONE + ): + return False + pending = self._pending + self._pending = _PendingEvent.NONE + try: + await asyncio.to_thread(self._flush, pending) + except Exception as exc: + with self._state_lock: + self._pending |= pending + logger.warning( + "Worker progress flush failed; retained for retry", + extra={ + "event": "worker_progress_flush_failed", + "error_code": f"worker_progress_error:{type(exc).__name__}", + "result": "retained", + }, + ) + return False + return True + + async def stop(self) -> bool: + """Stop periodic writes and atomically persist pending bits plus graceful stop.""" + + flush_stop = self._flush_stop + flush_task = self._flush_task + with self._state_lock: + if not self._registered or self._stopped: + return False + self._stopping = True + if flush_stop is not None: + flush_stop.set() + if flush_task is not None: + await flush_task + flush_lock = self._flush_lock + if flush_lock is None: + return False + async with flush_lock: + with self._state_lock: + if not self._registered or self._stopped: + return False + pending = self._pending + self._pending = _PendingEvent.NONE + try: + await asyncio.to_thread(self._stop, pending) + except Exception as exc: + with self._state_lock: + self._pending |= pending + logger.warning( + "Worker progress graceful stop failed; generation will become stale", + extra={ + "event": "worker_progress_stop_failed", + "error_code": f"worker_progress_error:{type(exc).__name__}", + "result": "stale", + }, + ) + return False + with self._state_lock: + self._stopped = True + return True + + async def _flush_loop(self) -> None: + stop = self._flush_stop + if stop is None: + return + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=self._flush_seconds) + except TimeoutError: + await self.flush_now() + + def _register(self) -> None: + with self._session_factory() as session, session.begin(): + now = database_utc_now(session) + session.add( + WorkerProcess( + generation_id=self.generation_id, + worker_id=self._worker_id, + started_at=now, + last_seen_at=now, + ) + ) + + def _flush(self, pending: _PendingEvent) -> None: + with self._session_factory() as session, session.begin(): + now = database_utc_now(session) + values = self._event_values(pending, now=now) + result = session.execute( + update(WorkerProcess) + .where( + WorkerProcess.generation_id == self.generation_id, + WorkerProcess.worker_id == self._worker_id, + WorkerProcess.stopped_at.is_(None), + ) + .values(last_seen_at=now, **values) + ) + if result.rowcount != 1: + raise WorkerProgressStateError("worker_progress_generation_not_active") + + def _stop(self, pending: _PendingEvent) -> None: + with self._session_factory() as session, session.begin(): + now = database_utc_now(session) + values: dict[str, datetime] = {"stopped_at": now} + if pending != _PendingEvent.NONE: + values["last_seen_at"] = now + values.update(self._event_values(pending, now=now)) + result = session.execute( + update(WorkerProcess) + .where( + WorkerProcess.generation_id == self.generation_id, + WorkerProcess.worker_id == self._worker_id, + WorkerProcess.stopped_at.is_(None), + ) + .values(**values) + ) + if result.rowcount != 1: + raise WorkerProgressStateError("worker_progress_generation_not_active") + + @staticmethod + def _event_values( + pending: _PendingEvent, + *, + now: datetime, + ) -> dict[str, datetime]: + values: dict[str, datetime] = {} + if pending & _PendingEvent.SCAN: + values["last_scan_at"] = now + if pending & _PendingEvent.CLAIM: + values["last_claim_at"] = now + if pending & _PendingEvent.PROGRESS: + values["last_progress_at"] = now + if pending & _PendingEvent.LEASE_HEARTBEAT: + values["last_lease_heartbeat_at"] = now + return values diff --git a/backend/app/workers/service.py b/backend/app/workers/service.py index 7968473..1353c4b 100644 --- a/backend/app/workers/service.py +++ b/backend/app/workers/service.py @@ -19,6 +19,11 @@ from app.runners.evaluation_runner import EvaluationRunner from app.runners.run_leases import ReapReport, RunLeaseRepository from app.task_queue import QueueUnavailable, RedisRunQueue, RunTaskDelivery +from app.worker_progress import ( + WorkerProgressEvent, + WorkerProgressLifecycle, + WorkerProgressRecorder, +) logger = logging.getLogger(__name__) @@ -40,6 +45,7 @@ def __init__( worker_id: str | None = None, lease_repository: RunLeaseRepository | None = None, runner: EvaluationRunner | None = None, + progress_recorder: WorkerProgressLifecycle | None = None, ) -> None: self._session_factory = session_factory self.worker_id = worker_id or default_worker_id() @@ -51,10 +57,16 @@ def __init__( retry_backoff_base=timedelta(seconds=settings.worker_retry_backoff_base_seconds), retry_backoff_cap=timedelta(seconds=settings.worker_retry_backoff_cap_seconds), ) + self._progress = progress_recorder or WorkerProgressRecorder( + session_factory, + worker_id=self.worker_id, + flush_seconds=settings.worker_progress_flush_seconds, + ) self._runner = runner or EvaluationRunner( session_factory, worker_id=self.worker_id, lease_repository=self._repository, + progress_observer=self._progress, ) self._queue_initialized = False self._queue_available: bool | None = None @@ -70,6 +82,7 @@ async def run_once(self) -> bool: run_ids = self._due_run_ids() if run_ids is None: return False + self._progress.note(WorkerProgressEvent.SCAN) if run_ids: await self._execute_run(run_ids[0]) return True @@ -84,6 +97,11 @@ async def run(self, stop: asyncio.Event) -> None: active: asyncio.Task[bool] | None = None active_delivery: RunTaskDelivery | None = None + await self._progress.start() + logger.info( + "Worker main loop registered and started", + extra={"event": "worker_main_loop_started", "worker_id": self.worker_id}, + ) try: while True: if active is not None: @@ -113,6 +131,8 @@ async def run(self, stop: asyncio.Event) -> None: continue run_ids = self._due_run_ids() + if run_ids is not None: + self._progress.note(WorkerProgressEvent.SCAN) await asyncio.sleep(0) if stop.is_set(): return @@ -172,6 +192,7 @@ async def run(self, stop: asyncio.Event) -> None: if active is not None and not active.done(): active.cancel() await asyncio.gather(active, return_exceptions=True) + await self._progress.stop() async def _next_delivery_until_stopped( self, @@ -451,6 +472,7 @@ def _reap_expired(self) -> ReapReport | None: return None self._database_recovered() if report.cancelled or report.dead_lettered or report.completed or report.retry_scheduled: + self._progress.note(WorkerProgressEvent.PROGRESS) logger.info( "Expired Run reconciliation completed", extra={ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 559090e..26ae97c 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ [project.scripts] llmbenchlab-evaluate = "app.cli.evaluate:main" +llmbenchlab-audit-retention = "app.cli.audit_retention:main" llmbenchlab-worker = "app.worker:main" llmbenchlab-worker-probe = "app.worker_probe:main" diff --git a/backend/tests/fixtures/audit-archive-v1-empty.jsonl b/backend/tests/fixtures/audit-archive-v1-empty.jsonl new file mode 100644 index 0000000..8d1ec36 --- /dev/null +++ b/backend/tests/fixtures/audit-archive-v1-empty.jsonl @@ -0,0 +1,2 @@ +{"cutoff_at":"2026-01-01T00:00:00.000000Z","event_limit":10000,"record_type":"header","schema":"llmbenchlab-audit-archive-v1","source_alembic_head":"20260828_0005"} +{"content_sha256":"a551a37348b9301e39f1c841364649642a4108cec2f3486b77c04c2c714ff02b","cutoff_at":"2026-01-01T00:00:00.000000Z","event_count":0,"event_type_counts":{},"expires_at_max":null,"expires_at_min":null,"has_more_eligible":false,"occurred_at_max":null,"occurred_at_min":null,"record_type":"manifest","retention_class_counts":{"operational":0,"security":0},"schema":"llmbenchlab-audit-archive-v1","source_alembic_head":"20260828_0005"} diff --git a/backend/tests/fixtures/audit-archive-v1-settled.jsonl b/backend/tests/fixtures/audit-archive-v1-settled.jsonl new file mode 100644 index 0000000..f8b2ff8 --- /dev/null +++ b/backend/tests/fixtures/audit-archive-v1-settled.jsonl @@ -0,0 +1,3 @@ +{"cutoff_at":"2026-01-01T00:00:00.000000Z","event_limit":10000,"record_type":"header","schema":"llmbenchlab-audit-archive-v1","source_alembic_head":"20260828_0005"} +{"attempt":null,"correlation_id":null,"duration_ms_hex":null,"event_key":"archive:provider-attempt:settled","event_type":"provider_attempt_settled","expires_at":"2025-04-01T12:00:00.000000Z","id":"00000000-0000-0000-0000-000000000099","lease_token":null,"model_id":null,"occurred_at":"2025-01-01T12:00:00.000000Z","payload":{"cost_usd":"1","disposition":"settled_actual","input_tokens":1,"outcome":"succeeded","output_tokens":1,"reconciled":false},"payload_hash":"7b042532fc04d25fdc802645a4957171122b7a0354eee0af6f97165ea0ff8f4e","provider_attempt":1,"question_id":null,"record_type":"audit_event","reservation_id":null,"retention_class":"operational","run_id":null,"worker_id":null} +{"content_sha256":"adf5294725e93f311eee1e43cd27a41aa56b401263a565968844066b4a59e616","cutoff_at":"2026-01-01T00:00:00.000000Z","event_count":1,"event_type_counts":{"provider_attempt_settled":1},"expires_at_max":"2025-04-01T12:00:00.000000Z","expires_at_min":"2025-04-01T12:00:00.000000Z","has_more_eligible":false,"occurred_at_max":"2025-01-01T12:00:00.000000Z","occurred_at_min":"2025-01-01T12:00:00.000000Z","record_type":"manifest","retention_class_counts":{"operational":1,"security":0},"schema":"llmbenchlab-audit-archive-v1","source_alembic_head":"20260828_0005"} diff --git a/backend/tests/integration/test_postgres_audit_retention.py b/backend/tests/integration/test_postgres_audit_retention.py new file mode 100644 index 0000000..0fffa16 --- /dev/null +++ b/backend/tests/integration/test_postgres_audit_retention.py @@ -0,0 +1,242 @@ +"""Real PostgreSQL audit-retention snapshot, advisory lock, and round trip.""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from queue import Queue +from time import monotonic, sleep + +import pytest +from sqlalchemy import func, select, text +from sqlalchemy.engine import make_url +from sqlalchemy.orm import Session, sessionmaker + +from app.db.clock import database_utc_now +from app.db.session import create_database_engine +from app.governance.audit import append_audit_event +from app.governance.audit_archive import verify_archive +from app.governance.audit_retention import ( + _POSTGRES_ADVISORY_LOCK, + _configure_mutation_transaction, + _reconcile_in_session, + archive_expired_events, + delete_archive_events, + reconcile_archive, + restore_archive, +) +from app.models import AuditEvent + +pytestmark = pytest.mark.integration + + +def _wait_for_ungranted_lock( + session_factory: sessionmaker[Session], + *, + backend_pid: int, + lock_kind: str, + timeout_seconds: float = 5.0, +) -> bool: + if lock_kind == "advisory": + statement = text( + "SELECT EXISTS (" + "SELECT 1 FROM pg_locks WHERE pid = :backend_pid AND granted IS FALSE " + "AND locktype = 'advisory' AND classid = :classid AND objid = :objid " + "AND objsubid = 2)" + ) + parameters = { + "backend_pid": backend_pid, + "classid": _POSTGRES_ADVISORY_LOCK[0], + "objid": _POSTGRES_ADVISORY_LOCK[1], + } + elif lock_kind == "row": + statement = text( + "SELECT EXISTS (" + "SELECT 1 FROM pg_locks WHERE pid = :backend_pid AND granted IS FALSE " + "AND locktype IN ('transactionid', 'tuple'))" + ) + parameters = {"backend_pid": backend_pid} + else: + raise ValueError(f"unsupported lock kind: {lock_kind}") + + deadline = monotonic() + timeout_seconds + with session_factory() as monitor: + while monotonic() < deadline: + if bool(monitor.scalar(statement, parameters)): + return True + sleep(0.01) + return False + + +@pytest.fixture +def postgres_retention_store(): + database_url = os.environ.get("LLMBENCHLAB_TEST_POSTGRES_URL") + if not database_url: + pytest.skip("LLMBENCHLAB_TEST_POSTGRES_URL is required") + url = make_url(database_url) + if url.get_backend_name() != "postgresql": + pytest.fail("LLMBENCHLAB_TEST_POSTGRES_URL must use PostgreSQL") + database_name = url.database or "" + if ( + "test" not in database_name.lower() + and os.environ.get("LLMBENCHLAB_TEST_ALLOW_TRUNCATE") != "1" + ): + pytest.fail( + "PostgreSQL integration tests require a database name containing 'test' or " + "LLMBENCHLAB_TEST_ALLOW_TRUNCATE=1" + ) + engine = create_database_engine(database_url) + factory = sessionmaker(bind=engine, class_=Session, expire_on_commit=False) + with engine.begin() as connection: + connection.execute(text("TRUNCATE audit_events RESTART IDENTITY CASCADE")) + try: + yield factory + finally: + with engine.begin() as connection: + connection.execute(text("TRUNCATE audit_events RESTART IDENTITY CASCADE")) + engine.dispose() + + +def test_postgres_mutation_advisory_lock_serializes_second_connection( + postgres_retention_store, +) -> None: + first = postgres_retention_store() + contender_pid: Queue[int] = Queue(maxsize=1) + + def contend_for_mutation_lock() -> int: + with postgres_retention_store() as session: + session.execute(text("SET LOCAL statement_timeout = '10s'")) + backend_pid = int(session.scalar(text("SELECT pg_backend_pid()"))) + contender_pid.put(backend_pid) + _configure_mutation_transaction(session) + session.rollback() + return backend_pid + + try: + _configure_mutation_transaction(first) + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(contend_for_mutation_lock) + backend_pid = contender_pid.get(timeout=5) + try: + assert _wait_for_ungranted_lock( + postgres_retention_store, + backend_pid=backend_pid, + lock_kind="advisory", + ) + assert not future.done() + finally: + first.rollback() + assert future.result(timeout=5) == backend_pid + finally: + first.rollback() + first.close() + + +def test_postgres_exact_reconcile_for_update_locks_audit_row( + postgres_retention_store, + tmp_path, +) -> None: + with postgres_retention_store() as session, session.begin(): + now = database_utc_now(session) + append_audit_event( + session, + event_key="pg-retention:row-lock", + event_type="run_claimed", + occurred_at=now - timedelta(days=91), + payload={"dispatch_count": 1}, + worker_id="pg-retention-worker", + attempt=1, + lease_token=1, + ) + + output = tmp_path / "postgres-retention-row-lock.jsonl" + written = archive_expired_events(postgres_retention_store, output) + archive = verify_archive(output, expected_sha256=written.archive_sha256) + assert len(archive.events) == 1 + event_id = archive.events[0].id + + first = postgres_retention_store() + contender_pid: Queue[int] = Queue(maxsize=1) + + def update_locked_row() -> tuple[int, int]: + with postgres_retention_store() as session: + session.execute(text("SET LOCAL statement_timeout = '10s'")) + backend_pid = int(session.scalar(text("SELECT pg_backend_pid()"))) + contender_pid.put(backend_pid) + result = session.execute( + text("UPDATE audit_events SET event_type = event_type WHERE id = :event_id"), + {"event_id": event_id}, + ) + rowcount = int(result.rowcount) + session.rollback() + return backend_pid, rowcount + + try: + report, exact_rows = _reconcile_in_session(first, archive, for_update=True) + assert report.status == "all_exact" + assert set(exact_rows) == {event_id} + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(update_locked_row) + backend_pid = contender_pid.get(timeout=5) + try: + assert _wait_for_ungranted_lock( + postgres_retention_store, + backend_pid=backend_pid, + lock_kind="row", + ) + assert not future.done() + finally: + first.rollback() + assert future.result(timeout=5) == (backend_pid, 1) + finally: + first.rollback() + first.close() + + +def test_postgres_archive_delete_reconcile_restore_round_trip( + postgres_retention_store, + tmp_path, +) -> None: + with postgres_retention_store() as session, session.begin(): + now = database_utc_now(session) + append_audit_event( + session, + event_key="pg-retention:expired", + event_type="run_claimed", + occurred_at=now - timedelta(days=91), + payload={"dispatch_count": 1}, + worker_id="pg-retention-worker", + attempt=1, + lease_token=1, + ) + append_audit_event( + session, + event_key="pg-retention:future", + event_type="run_claimed", + occurred_at=now, + payload={"dispatch_count": 2}, + worker_id="pg-retention-worker", + attempt=1, + lease_token=1, + ) + + output = tmp_path / "postgres-retention.jsonl" + written = archive_expired_events(postgres_retention_store, output) + archive = verify_archive(output, expected_sha256=written.archive_sha256) + assert len(archive.events) == 1 + assert archive.events[0].event_key == "pg-retention:expired" + assert reconcile_archive(postgres_retention_store, archive).status == "all_exact" + + deleted = delete_archive_events(postgres_retention_store, archive) + assert deleted.status == "deleted" + assert deleted.changed_count == 1 + assert reconcile_archive(postgres_retention_store, archive).status == "all_absent" + with postgres_retention_store() as session: + assert session.scalar(select(func.count(AuditEvent.id))) == 1 + + restored = restore_archive(postgres_retention_store, archive) + assert restored.status == "restored" + assert reconcile_archive(postgres_retention_store, archive).status == "all_exact" + with postgres_retention_store() as session: + assert session.scalar(select(func.count(AuditEvent.id))) == 2 diff --git a/backend/tests/integration/test_postgres_leases.py b/backend/tests/integration/test_postgres_leases.py index 3f07832..8f3059a 100644 --- a/backend/tests/integration/test_postgres_leases.py +++ b/backend/tests/integration/test_postgres_leases.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os import time from concurrent.futures import ThreadPoolExecutor @@ -17,6 +18,7 @@ from app.adapters.base import ProviderAttemptDisposition, ProviderAttemptOutcome from app.api.v1.models import _active_run_exists +from app.db.clock import database_utc_now from app.db.session import create_database_engine from app.governance import ( AuditIntegrityError, @@ -44,8 +46,14 @@ ProviderType, Question, RunStatus, + WorkerProcess, ) from app.runners.run_leases import CancelDisposition, ResponseDisposition, RunLeaseRepository +from app.worker_progress import ( + WorkerProgressEvent, + WorkerProgressRecorder, + collect_worker_progress, +) pytestmark = pytest.mark.integration @@ -71,7 +79,8 @@ def postgres_store(): with engine.begin() as connection: connection.execute( text( - "TRUNCATE audit_events, provider_call_reservations, question_executions, " + "TRUNCATE worker_processes, audit_events, provider_call_reservations, " + "question_executions, " "governance_minute_buckets, governance_scopes, evaluation_responses, " "evaluation_runs, questions, benchmarks, models, governance_policies " "RESTART IDENTITY CASCADE" @@ -126,6 +135,48 @@ def postgres_store(): engine.dispose() +@pytest.mark.asyncio +async def test_postgres_worker_generations_flush_and_stop_without_cross_talk( + postgres_store, +) -> None: + recorders = [ + WorkerProgressRecorder( + postgres_store, + worker_id=f"worker-progress-pg-{index}", + flush_seconds=60, + ) + for index in range(2) + ] + await asyncio.gather(*(recorder.start() for recorder in recorders)) + recorders[0].note(WorkerProgressEvent.SCAN) + recorders[1].note(WorkerProgressEvent.PROGRESS) + assert await asyncio.gather(*(recorder.flush_now() for recorder in recorders)) == [True, True] + + with postgres_store() as session: + now = database_utc_now(session) + snapshot = collect_worker_progress( + session, + now=now, + stale_seconds=60, + expected=2, + ) + assert snapshot.registered == 2 + assert snapshot.live == 2 + assert snapshot.stalled == 0 + assert snapshot.shortfall == 0 + + assert await asyncio.gather(*(recorder.stop() for recorder in recorders)) == [True, True] + with postgres_store() as session: + assert ( + session.scalar( + select(func.count(WorkerProcess.generation_id)).where( + WorkerProcess.stopped_at.is_not(None) + ) + ) + == 2 + ) + + def _response() -> EvaluationResponse: return EvaluationResponse( run_id="run-pg", diff --git a/backend/tests/integration/test_redis_streams.py b/backend/tests/integration/test_redis_streams.py index ccf88ce..c172e63 100644 --- a/backend/tests/integration/test_redis_streams.py +++ b/backend/tests/integration/test_redis_streams.py @@ -44,10 +44,12 @@ async def test_real_redis_pending_delivery_is_autoclaimed_and_acked() -> None: version = tuple(int(part) for part in server["redis_version"].split(".")[:2]) assert version >= (6, 2) - await queue.publish("run-real-redis", correlation_id="correlation-real-redis") + run_id = str(uuid4()) + correlation_id = str(uuid4()) + await queue.publish(run_id, correlation_id=correlation_id) await queue.ensure_consumer_group() first = await queue.read_new(consumer="worker-a", block_milliseconds=100) - assert first is not None and first.run_id == "run-real-redis" + assert first is not None and first.run_id == run_id cursor, claimed = await queue.claim_stale( consumer="worker-b", @@ -179,7 +181,7 @@ async def test_real_worker_commits_protocol_evidence_before_ack_and_duplicate_is ) assert created.status_code == 202 run_id = created.json()["id"] - await queue.publish(run_id, correlation_id="queue-first-delivery") + await queue.publish(run_id, correlation_id=str(uuid4())) settings = get_settings() repository = _QueueFirstRepository( SessionLocal, @@ -207,7 +209,7 @@ async def test_real_worker_commits_protocol_evidence_before_ack_and_duplicate_is assert float(durable["estimated_cost"] or 0) == 0 assert (await raw.xpending(stream, group))["pending"] == 0 - await ambiguous_queue.publish(run_id, correlation_id="duplicate-after-unknown-ack") + await ambiguous_queue.publish(run_id, correlation_id=str(uuid4())) assert await worker.run_once() is True assert _run_snapshot(run_id) == durable diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 20aa5e9..35f15e9 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -241,6 +241,20 @@ def test_request_log_uses_route_template_instead_of_unmatched_user_path(client, assert "sk-secret-value" not in completion.getMessage() +def test_request_log_normalizes_attacker_selected_http_method(client, caplog) -> None: + secret_method = "SK-HTTP-METHOD-CANARY" + caplog.set_level("INFO", logger="app.main") + + response = client.request(secret_method, "/api/v1/live") + + assert response.status_code == 405 + completion = next( + record for record in reversed(caplog.records) if record.event == "api_request_completed" + ) + assert completion.request_method == "unsupported" + assert secret_method not in completion.getMessage() + + def test_task_metrics_are_database_derived_and_do_not_replace_dashboard_metrics(client) -> None: empty = client.get("/api/v1/tasks/metrics") assert empty.status_code == 200 @@ -261,6 +275,17 @@ def test_task_metrics_are_database_derived_and_do_not_replace_dashboard_metrics( "total_attempts": 0, "total_failed_attempts": 0, "total_dispatches": 0, + "worker_expected_processes": 1, + "worker_registered_processes": 0, + "worker_live_processes": 0, + "worker_stalled_processes": 0, + "worker_shortfall_processes": 1, + "worker_stale_after_seconds": 60.0, + "worker_last_seen_at": None, + "worker_last_scan_at": None, + "worker_last_claim_at": None, + "worker_last_progress_at": None, + "worker_last_lease_heartbeat_at": None, } model = client.post( "/api/v1/models", diff --git a/backend/tests/test_audit_api.py b/backend/tests/test_audit_api.py index 74a7f42..ce6436f 100644 --- a/backend/tests/test_audit_api.py +++ b/backend/tests/test_audit_api.py @@ -2,6 +2,7 @@ from datetime import UTC, datetime, timedelta +import pytest from sqlalchemy import select from app.governance.audit import append_audit_event @@ -149,6 +150,104 @@ def test_run_audit_fails_closed_without_reflecting_a_corrupt_payload( assert secret not in caplog.text +@pytest.mark.parametrize("damage", ["payload_json", "occurred_at"]) +def test_run_audit_classifies_invalid_stored_value_as_integrity_error( + client, + db_session, + caplog, + damage, +) -> None: + marker = "sk-invalid-run-audit-json-never-reflect" + model = client.post( + "/api/v1/models", + json={"name": "Invalid JSON Audit Mock", "provider_type": "mock", "enabled": True}, + ).json() + benchmark = client.post("/api/v1/benchmarks/reload-demo").json() + run = client.post( + "/api/v1/runs", + json={"model_id": model["id"], "benchmark_id": benchmark["id"]}, + ).json() + occurred_at = datetime(2026, 8, 27, 17, 0, tzinfo=UTC) + event = append_audit_event( + db_session, + event_key=f"audit-api:{run['id']}:invalid-json", + event_type="run_claimed", + occurred_at=occurred_at, + payload={"dispatch_count": 1}, + run_id=run["id"], + worker_id="worker-audit-api", + attempt=1, + lease_token=1, + ) + event_id = event.id + db_session.commit() + if damage == "payload_json": + db_session.connection().exec_driver_sql( + "UPDATE audit_events SET payload = ? WHERE id = ?", + ("{" + marker, event_id), + ) + else: + malformed_timestamp = occurred_at.replace(tzinfo=None).isoformat(sep=" ") + marker + db_session.connection().exec_driver_sql( + "UPDATE audit_events SET occurred_at = ? WHERE id = ?", + (malformed_timestamp, event_id), + ) + db_session.commit() + caplog.set_level("ERROR", logger="app.api.v1.runs") + + response = client.get(f"/api/v1/runs/{run['id']}/audit") + + assert response.status_code == 500 + assert response.json() == { + "detail": { + "code": "audit_event_integrity_error", + "message": "A retained audit event failed integrity validation", + } + } + assert marker not in response.text + assert marker not in caplog.text + + +def test_run_audit_rejects_noncanonical_stored_payload_representation( + client, + db_session, +) -> None: + model = client.post( + "/api/v1/models", + json={"name": "Canonical Audit Mock", "provider_type": "mock", "enabled": True}, + ).json() + benchmark = client.post("/api/v1/benchmarks/reload-demo").json() + run = client.post( + "/api/v1/runs", + json={"model_id": model["id"], "benchmark_id": benchmark["id"]}, + ).json() + event = append_audit_event( + db_session, + event_key=f"audit-api:{run['id']}:payload-canonical", + event_type="provider_attempt_settled", + occurred_at=datetime(2026, 8, 27, 17, 0, tzinfo=UTC), + payload={ + "disposition": "settled_actual", + "outcome": "succeeded", + "input_tokens": 1, + "output_tokens": 1, + "cost_usd": "1", + "reconciled": False, + }, + correlation_id=run["id"], + run_id=run["id"], + model_id=model["id"], + provider_attempt=1, + ) + event.payload["cost_usd"] = 1 + db_session.commit() + + response = client.get(f"/api/v1/runs/{run['id']}/audit") + + assert response.status_code == 500 + assert response.json()["detail"]["code"] == "audit_event_integrity_error" + + def test_run_audit_rejects_nonfinite_retained_duration(client, db_session) -> None: model = client.post( "/api/v1/models", @@ -177,3 +276,34 @@ def test_run_audit_rejects_nonfinite_retained_duration(client, db_session) -> No assert response.status_code == 500 assert response.json()["detail"]["code"] == "audit_event_integrity_error" + + +def test_run_audit_validates_non_response_storage_identities(client, db_session, caplog) -> None: + marker = "https://secret.example.invalid/audit-event-key" + model = client.post( + "/api/v1/models", + json={"name": "Identity Audit Mock", "provider_type": "mock", "enabled": True}, + ).json() + benchmark = client.post("/api/v1/benchmarks/reload-demo").json() + run = client.post( + "/api/v1/runs", + json={"model_id": model["id"], "benchmark_id": benchmark["id"]}, + ).json() + event = append_audit_event( + db_session, + event_key=f"audit-api:{run['id']}:identity", + event_type="run_claimed", + occurred_at=datetime(2026, 8, 27, 17, 0, tzinfo=UTC), + payload={"dispatch_count": 1}, + run_id=run["id"], + ) + event.event_key = marker + db_session.commit() + caplog.set_level("ERROR", logger="app.api.v1.runs") + + response = client.get(f"/api/v1/runs/{run['id']}/audit") + + assert response.status_code == 500 + assert response.json()["detail"]["code"] == "audit_event_integrity_error" + assert marker not in response.text + assert marker not in caplog.text diff --git a/backend/tests/test_audit_archive.py b/backend/tests/test_audit_archive.py new file mode 100644 index 0000000..33abcca --- /dev/null +++ b/backend/tests/test_audit_archive.py @@ -0,0 +1,316 @@ +"""Canonical audit archive format and filesystem boundary tests.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +import app.governance.audit_archive as audit_archive_module +from app.governance.audit import validate_audit_event_values_for_read +from app.governance.audit_archive import ( + ARCHIVE_EVENT_LIMIT, + AuditArchiveError, + build_archive_bytes, + verify_archive, + write_archive, +) +from app.models import AuditRetentionClass + + +def _facts(index: int = 1): + occurred_at = datetime(2025, 1, index, 12, 0, tzinfo=UTC) + payload = {"dispatch_count": index} + encoded = json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + return validate_audit_event_values_for_read( + id=f"00000000-0000-0000-0000-{index:012d}", + event_key=f"archive:run:{index}:claimed", + event_type="run_claimed", + payload_hash=hashlib.sha256(encoded.encode()).hexdigest(), + payload=payload, + retention_class=AuditRetentionClass.OPERATIONAL, + occurred_at=occurred_at, + expires_at=occurred_at + timedelta(days=90), + correlation_id=f"run-{index}", + run_id=f"run-{index}", + model_id=f"model-{index}", + question_id=None, + worker_id=f"worker-{index}", + reservation_id=None, + attempt=0, + provider_attempt=None, + lease_token=index, + duration_ms=-0.0, + ) + + +def _settled_facts(): + occurred_at = datetime(2025, 1, 1, 12, 0, tzinfo=UTC) + payload = { + "disposition": "settled_actual", + "outcome": "succeeded", + "input_tokens": 1, + "output_tokens": 1, + "cost_usd": "1", + "reconciled": False, + } + encoded = json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + return validate_audit_event_values_for_read( + id="00000000-0000-0000-0000-000000000099", + event_key="archive:provider-attempt:settled", + event_type="provider_attempt_settled", + payload_hash=hashlib.sha256(encoded.encode()).hexdigest(), + payload=payload, + retention_class=AuditRetentionClass.OPERATIONAL, + occurred_at=occurred_at, + expires_at=occurred_at + timedelta(days=90), + correlation_id=None, + run_id=None, + model_id=None, + question_id=None, + worker_id=None, + reservation_id=None, + attempt=None, + provider_attempt=1, + lease_token=None, + duration_ms=None, + ) + + +def test_archive_round_trip_is_canonical_private_and_no_replace(tmp_path) -> None: + path = tmp_path / "audit.jsonl" + events = (_facts(1), _facts(2)) + result = write_archive( + path, + events, + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=True, + ) + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert path.read_bytes().endswith(b"\n") + assert b'"duration_ms_hex":"0x0.0p+0"' in path.read_bytes() + verified = verify_archive(path, expected_sha256=result.archive_sha256) + assert verified.events == events + assert verified.has_more_eligible is True + assert verified.content_sha256 == result.content_sha256 + assert verified.archive_sha256 == result.archive_sha256 + + with pytest.raises(AuditArchiveError, match="archive_output_exists"): + write_archive( + path, + events, + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + +def test_archive_rejects_broad_permissions_symlink_and_digest_mismatch(tmp_path) -> None: + path = tmp_path / "audit.jsonl" + result = write_archive( + path, + (_facts(),), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + os.chmod(path, 0o640) + with pytest.raises(AuditArchiveError, match="archive_input_permissions_too_broad"): + verify_archive(path) + + os.chmod(path, 0o600) + with pytest.raises(AuditArchiveError, match="archive_digest_mismatch"): + verify_archive(path, expected_sha256="0" * 64) + assert verify_archive(path, expected_sha256=result.archive_sha256).events + + link = tmp_path / "link.jsonl" + link.symlink_to(path) + with pytest.raises(AuditArchiveError, match="archive_input_open_failed"): + verify_archive(link) + + +def test_archive_output_rejects_symlink_unowned_or_writable_parent( + tmp_path, + monkeypatch, +) -> None: + private_parent = tmp_path / "private" + private_parent.mkdir(mode=0o700) + linked_parent = tmp_path / "linked" + linked_parent.symlink_to(private_parent, target_is_directory=True) + with pytest.raises(AuditArchiveError, match="archive_output_parent_invalid"): + write_archive( + linked_parent / "audit.jsonl", + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + private_parent.chmod(0o720) + with pytest.raises( + AuditArchiveError, + match="archive_output_parent_permissions_too_broad", + ): + write_archive( + private_parent / "audit.jsonl", + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + private_parent.chmod(0o700) + + monkeypatch.setattr(os, "geteuid", lambda: os.getuid() + 1) + with pytest.raises(AuditArchiveError, match="archive_output_parent_not_owned"): + write_archive( + private_parent / "audit.jsonl", + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + +def test_archive_rejects_duplicate_json_keys_and_noncanonical_records(tmp_path) -> None: + duplicate = tmp_path / "duplicate.jsonl" + duplicate.write_bytes( + b'{"record_type":"header","record_type":"header"}\n{"record_type":"manifest"}\n' + ) + os.chmod(duplicate, 0o600) + with pytest.raises(AuditArchiveError, match="archive_json_duplicate_key"): + verify_archive(duplicate) + + noncanonical = tmp_path / "noncanonical.jsonl" + data, _digest = build_archive_bytes( + (_facts(),), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + noncanonical.write_bytes(data.replace(b'{"cutoff_at"', b'{ "cutoff_at"', 1)) + os.chmod(noncanonical, 0o600) + with pytest.raises(AuditArchiveError, match="archive_record_not_canonical"): + verify_archive(noncanonical) + + +def test_archive_enforces_event_limit_before_materializing_output() -> None: + with pytest.raises(AuditArchiveError, match="archive_event_limit_exceeded"): + build_archive_bytes( + (_facts(),) * (ARCHIVE_EVENT_LIMIT + 1), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + +def test_archive_rejects_excess_record_count_before_decoding(tmp_path, monkeypatch) -> None: + path = tmp_path / "too-many-small-records.jsonl" + path.write_bytes(b"{}\n" * (ARCHIVE_EVENT_LIMIT + 3)) + path.chmod(0o600) + + monkeypatch.setattr( + audit_archive_module, + "_decode_line", + lambda _line: pytest.fail("over-limit archive reached JSON decoding"), + ) + + with pytest.raises(AuditArchiveError, match="archive_event_limit_exceeded"): + verify_archive(path) + + +def test_manifest_counts_reject_boolean_number_confusion(tmp_path) -> None: + data, _digest = build_archive_bytes( + (_facts(),), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + lines = data.rstrip(b"\n").split(b"\n") + manifest = json.loads(lines[-1]) + manifest["event_count"] = True + lines[-1] = json.dumps( + manifest, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode() + path = tmp_path / "bool-count.jsonl" + path.write_bytes(b"\n".join(lines) + b"\n") + path.chmod(0o600) + with pytest.raises(AuditArchiveError, match="archive_manifest_rollup_invalid"): + verify_archive(path) + + +def test_archive_rejects_noncanonical_raw_payload_even_when_normalized_hash_matches() -> None: + canonical = _settled_facts() + drifted = replace( + canonical, + payload={**canonical.payload, "cost_usd": 1}, + ) + + with pytest.raises(AuditArchiveError, match="archive_event_integrity_invalid"): + build_archive_bytes( + (drifted,), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + +def test_archive_v1_rejects_unsupported_source_revision_on_write_and_read(tmp_path) -> None: + with pytest.raises(AuditArchiveError, match="archive_source_revision_unsupported"): + build_archive_bytes( + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="future_9999", + has_more_eligible=False, + ) + + fixture = Path(__file__).with_name("fixtures") / "audit-archive-v1-empty.jsonl" + path = tmp_path / "unsupported-source.jsonl" + path.write_bytes(fixture.read_bytes().replace(b"20260828_0005", b"future_9999")) + path.chmod(0o600) + with pytest.raises(AuditArchiveError, match="archive_source_revision_unsupported"): + verify_archive(path) + + +def test_frozen_archive_v1_fixture_remains_verifiable(tmp_path) -> None: + fixture = Path(__file__).with_name("fixtures") / "audit-archive-v1-empty.jsonl" + path = tmp_path / "archive-v1-empty.jsonl" + path.write_bytes(fixture.read_bytes()) + path.chmod(0o600) + + verified = verify_archive( + path, + expected_sha256="6c0d95198ea8767787e2572b55b3acbfaa3e165a57760c18d8f183254f72fa3d", + ) + + assert verified.source_alembic_head == "20260828_0005" + assert verified.events == () + assert verified.content_sha256 == ( + "a551a37348b9301e39f1c841364649642a4108cec2f3486b77c04c2c714ff02b" + ) + + settled_fixture = Path(__file__).with_name("fixtures") / "audit-archive-v1-settled.jsonl" + settled_path = tmp_path / "archive-v1-settled.jsonl" + settled_path.write_bytes(settled_fixture.read_bytes()) + settled_path.chmod(0o600) + settled = verify_archive( + settled_path, + expected_sha256="f3cf9e3484de84cd318295315b3210c66ecf6c66d33076d069a2c3ba2f1e36d8", + ) + + assert len(settled.events) == 1 + assert settled.events[0].event_type == "provider_attempt_settled" + assert settled.events[0].payload["cost_usd"] == "1" + assert settled.content_sha256 == ( + "adf5294725e93f311eee1e43cd27a41aa56b401263a565968844066b4a59e616" + ) diff --git a/backend/tests/test_audit_retention.py b/backend/tests/test_audit_retention.py new file mode 100644 index 0000000..671f7a0 --- /dev/null +++ b/backend/tests/test_audit_retention.py @@ -0,0 +1,335 @@ +"""SQLite audit retention archive/delete/reconcile/restore behavior.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import delete, select, text, update +from sqlalchemy.orm import Session + +from app.db.clock import database_utc_now +from app.db.session import SessionLocal +from app.governance.audit import append_audit_event, validate_audit_event_values_for_read +from app.governance.audit_archive import verify_archive, write_archive +from app.governance.audit_retention import ( + AuditRetentionCommitOutcomeUnknownError, + AuditRetentionCommittedVerificationError, + AuditRetentionError, + archive_expired_events, + delete_archive_events, + reconcile_archive, + restore_archive, +) +from app.models import AuditEvent, AuditRetentionClass + + +def _append_expired(session, *, index: int) -> None: + now = database_utc_now(session) + append_audit_event( + session, + event_key=f"retention:run:{index}:claimed", + event_type="run_claimed", + occurred_at=now - timedelta(days=91, minutes=index), + payload={"dispatch_count": index}, + correlation_id=f"retention-run-{index}", + run_id=f"retention-run-{index}", + model_id=f"retention-model-{index}", + worker_id=f"retention-worker-{index}", + attempt=index, + lease_token=index, + ) + + +def test_sqlite_archive_delete_reconcile_restore_round_trip( + client, + db_session, + tmp_path, +) -> None: + del client + _append_expired(db_session, index=1) + _append_expired(db_session, index=2) + db_session.commit() + + output = tmp_path / "retention.jsonl" + written = archive_expired_events(SessionLocal, output) + archive = verify_archive(output, expected_sha256=written.archive_sha256) + assert len(archive.events) == 2 + assert reconcile_archive(SessionLocal, archive).status == "all_exact" + + # An independently appended but already expired row is intentionally not + # in this archive. Exact delete must preserve it instead of deleting by + # the archive cutoff. + with SessionLocal() as session, session.begin(): + _append_expired(session, index=3) + + deleted = delete_archive_events(SessionLocal, archive) + assert deleted.status == "deleted" + assert deleted.changed_count == 2 + assert reconcile_archive(SessionLocal, archive).status == "all_absent" + with SessionLocal() as session: + assert ( + session.scalar( + select(AuditEvent.event_key).where( + AuditEvent.event_key == "retention:run:3:claimed" + ) + ) + == "retention:run:3:claimed" + ) + assert delete_archive_events(SessionLocal, archive).status == "already_absent" + + restored = restore_archive(SessionLocal, archive) + assert restored.status == "restored" + assert restored.changed_count == 2 + assert reconcile_archive(SessionLocal, archive).status == "all_exact" + assert restore_archive(SessionLocal, archive).status == "already_exact" + + +def test_delete_rejects_mixed_missing_and_conflicting_storage_facts( + client, + db_session, + tmp_path, +) -> None: + del client + _append_expired(db_session, index=1) + _append_expired(db_session, index=2) + db_session.commit() + output = tmp_path / "mixed.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + + with SessionLocal() as session, session.begin(): + session.execute(delete(AuditEvent).where(AuditEvent.id == archive.events[0].id)) + assert reconcile_archive(SessionLocal, archive).status == "mixed_exact_absent" + with pytest.raises(AuditRetentionError, match="audit_retention_delete_conflict"): + delete_archive_events(SessionLocal, archive) + + with SessionLocal() as session, session.begin(): + session.execute( + update(AuditEvent) + .where(AuditEvent.id == archive.events[1].id) + .values(event_key="retention:conflicting:key") + ) + assert reconcile_archive(SessionLocal, archive).status == "conflict" + with pytest.raises(AuditRetentionError, match="audit_retention_restore_conflict"): + restore_archive(SessionLocal, archive) + + +def test_delete_rolls_back_when_database_reports_a_partial_exact_delete( + client, + db_session, + tmp_path, +) -> None: + del client + _append_expired(db_session, index=1) + _append_expired(db_session, index=2) + db_session.commit() + output = tmp_path / "partial-delete.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + + with SessionLocal() as session, session.begin(): + session.execute( + text( + "CREATE TRIGGER ignore_one_audit_delete " + "BEFORE DELETE ON audit_events " + "WHEN OLD.event_key = 'retention:run:1:claimed' " + "BEGIN SELECT RAISE(IGNORE); END" + ) + ) + try: + with pytest.raises(AuditRetentionError, match="audit_retention_delete_count_mismatch"): + delete_archive_events(SessionLocal, archive) + assert reconcile_archive(SessionLocal, archive).status == "all_exact" + finally: + with SessionLocal() as session, session.begin(): + session.execute(text("DROP TRIGGER IF EXISTS ignore_one_audit_delete")) + + +def test_restore_requires_referenced_reservation(client, tmp_path) -> None: + del client + occurred = datetime(2025, 1, 1, tzinfo=UTC) + payload: dict[str, object] = {} + encoded = json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + event = validate_audit_event_values_for_read( + id="00000000-0000-0000-0000-000000000099", + event_key="retention:reservation:missing", + event_type="provider_attempt_send_started", + payload_hash=hashlib.sha256(encoded.encode()).hexdigest(), + payload=payload, + retention_class=AuditRetentionClass.OPERATIONAL, + occurred_at=occurred, + expires_at=occurred + timedelta(days=90), + correlation_id=None, + run_id=None, + model_id=None, + question_id=None, + worker_id=None, + reservation_id="00000000-0000-0000-0000-000000000001", + attempt=None, + provider_attempt=1, + lease_token=None, + duration_ms=None, + ) + output = tmp_path / "missing-reservation.jsonl" + write_archive( + output, + (event,), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + archive = verify_archive(output) + with pytest.raises(AuditRetentionError, match="audit_retention_reservation_missing"): + restore_archive(SessionLocal, archive) + + +def test_restore_reports_commit_outcome_unknown_and_reconcile_is_safe( + client, + db_session, + tmp_path, + monkeypatch, +) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + output = tmp_path / "unknown.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + delete_archive_events(SessionLocal, archive) + + original_commit = Session.commit + + def commit_then_raise(session: Session) -> None: + original_commit(session) + raise RuntimeError("simulated_ack_loss") + + monkeypatch.setattr(Session, "commit", commit_then_raise) + with pytest.raises(AuditRetentionCommitOutcomeUnknownError): + restore_archive(SessionLocal, archive) + assert reconcile_archive(SessionLocal, archive).status == "all_exact" + + +def test_same_key_with_different_id_is_a_conflict(client, db_session, tmp_path) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + output = tmp_path / "same-key.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + with SessionLocal() as session, session.begin(): + session.execute( + update(AuditEvent) + .where(AuditEvent.id == archive.events[0].id) + .values(id="00000000-0000-0000-0000-000000000777") + ) + assert reconcile_archive(SessionLocal, archive).status == "conflict" + with pytest.raises(AuditRetentionError, match="audit_retention_restore_conflict"): + restore_archive(SessionLocal, archive) + + +def test_postcommit_verification_failure_is_distinct( + client, + db_session, + tmp_path, + monkeypatch, +) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + output = tmp_path / "postcommit.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + + def fail_postcommit(_factory, _archive): + raise AuditRetentionCommittedVerificationError("simulated_postcommit_failure") + + monkeypatch.setattr("app.governance.audit_retention._postcommit_report", fail_postcommit) + with pytest.raises(AuditRetentionCommittedVerificationError): + delete_archive_events(SessionLocal, archive) + assert reconcile_archive(SessionLocal, archive).status == "all_absent" + + +def test_noop_restore_and_delete_still_run_independent_postverification( + client, + db_session, + tmp_path, + monkeypatch, +) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + output = tmp_path / "noop-postverify.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + + real_postcommit = __import__( + "app.governance.audit_retention", fromlist=["_postcommit_report"] + )._postcommit_report + observed_statuses: list[str] = [] + + def record_postcommit(factory, verified_archive): + report = real_postcommit(factory, verified_archive) + observed_statuses.append(report.status) + return report + + monkeypatch.setattr("app.governance.audit_retention._postcommit_report", record_postcommit) + assert restore_archive(SessionLocal, archive).status == "already_exact" + delete_archive_events(SessionLocal, archive) + assert delete_archive_events(SessionLocal, archive).status == "already_absent" + assert observed_statuses == ["all_exact", "all_absent", "all_absent"] + + +def test_commit_confirmed_cleanup_failure_is_exit_three_class( + client, + db_session, + tmp_path, + monkeypatch, +) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + output = tmp_path / "cleanup.jsonl" + archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + original_close = Session.close + + def close_then_raise(session: Session) -> None: + original_close(session) + raise RuntimeError("simulated_close_failure") + + with monkeypatch.context() as patch: + patch.setattr(Session, "close", close_then_raise) + with pytest.raises(AuditRetentionCommittedVerificationError): + delete_archive_events(SessionLocal, archive) + assert reconcile_archive(SessionLocal, archive).status == "all_absent" + + +def test_archive_hard_cap_sets_has_more(client, db_session, tmp_path, monkeypatch) -> None: + del client + _append_expired(db_session, index=1) + _append_expired(db_session, index=2) + db_session.commit() + monkeypatch.setattr("app.governance.audit_retention.ARCHIVE_EVENT_LIMIT", 1) + output = tmp_path / "bounded.jsonl" + result = archive_expired_events(SessionLocal, output) + archive = verify_archive(output) + assert result.event_count == 1 + assert result.has_more_eligible is True + assert archive.has_more_eligible is True + assert len(archive.events) == 1 + + +def test_public_full_row_projection_rejects_corrupt_duration(client, db_session) -> None: + del client + _append_expired(db_session, index=1) + db_session.commit() + event = db_session.scalar(select(AuditEvent)) + assert event is not None + event.duration_ms = float("inf") + from app.governance.audit import AuditIntegrityError, validate_audit_event_for_read + + with pytest.raises(AuditIntegrityError): + validate_audit_event_for_read(event) diff --git a/backend/tests/test_audit_retention_cli.py b/backend/tests/test_audit_retention_cli.py new file mode 100644 index 0000000..27fbf16 --- /dev/null +++ b/backend/tests/test_audit_retention_cli.py @@ -0,0 +1,207 @@ +"""Fixed, secret-minimized audit retention CLI behavior.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path + +from app.cli import audit_retention +from app.governance.audit_archive import write_archive + + +def test_verify_is_offline_and_does_not_print_archive_path(tmp_path, monkeypatch, capsys) -> None: + path = tmp_path / "secret-internal-name.jsonl" + written = write_archive( + path, + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + + def reject_database_access(): + raise AssertionError("verify must not create a database factory") + + monkeypatch.setattr(audit_retention, "_database_factory", reject_database_access) + status = audit_retention.main( + ["verify", "--archive", str(path), "--expected-sha256", written.archive_sha256] + ) + captured = capsys.readouterr() + assert status == 0 + assert "status=verified" in captured.out + assert written.archive_sha256 in captured.out + assert str(path) not in captured.out + assert captured.err == "" + + +def test_verify_failure_uses_fixed_code_without_path_or_file_content(tmp_path, capsys) -> None: + path = tmp_path / "private-path-marker.jsonl" + path.write_text("payload-secret-marker", encoding="utf-8") + path.chmod(0o600) + status = audit_retention.main(["verify", "--archive", str(path)]) + captured = capsys.readouterr() + assert status == 2 + assert "status=failed operation=verify" in captured.err + assert "private-path-marker" not in captured.err + assert "payload-secret-marker" not in captured.err + + +def test_cli_maps_commit_outcome_and_postcommit_failures(monkeypatch, capsys) -> None: + from app.governance.audit_retention import ( + AuditRetentionCommitOutcomeUnknownError, + AuditRetentionCommittedVerificationError, + ) + + def commit_unknown(_arguments): + raise AuditRetentionCommitOutcomeUnknownError( + "hidden", + event_count=7, + archive_sha256="a" * 64, + ) + + monkeypatch.setattr(audit_retention, "_run", commit_unknown) + assert audit_retention.main(["verify", "--archive", "ignored"]) == 4 + commit_output = capsys.readouterr().err + assert "status=commit_outcome_unknown" in commit_output + assert "count=7" in commit_output + assert f"archive_sha256={'a' * 64}" in commit_output + + def postcommit_failed(_arguments): + raise AuditRetentionCommittedVerificationError( + "hidden", + event_count=8, + archive_sha256="b" * 64, + ) + + monkeypatch.setattr(audit_retention, "_run", postcommit_failed) + assert audit_retention.main(["verify", "--archive", "ignored"]) == 3 + postcommit_output = capsys.readouterr().err + assert "status=committed_but_verification_failed" in postcommit_output + assert "count=8" in postcommit_output + assert f"archive_sha256={'b' * 64}" in postcommit_output + + +def test_verify_fresh_process_ignores_invalid_database_configuration( + tmp_path, +) -> None: + archive_path = tmp_path / "offline.jsonl" + written = write_archive( + archive_path, + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + backend_root = Path(__file__).resolve().parents[1] + environment = os.environ.copy() + environment.pop("DATABASE_URL", None) + environment["LLMBENCHLAB_DATABASE_URL"] = "not-a-sqlalchemy-url" + + completed = subprocess.run( + [ + sys.executable, + "-m", + "app.cli.audit_retention", + "verify", + "--archive", + str(archive_path), + "--expected-sha256", + written.archive_sha256, + ], + cwd=backend_root, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + + assert completed.returncode == 0 + assert "status=verified operation=verify count=0" in completed.stdout + assert "not-a-sqlalchemy-url" not in completed.stdout + completed.stderr + assert completed.stderr == "" + + +def test_verify_fresh_process_does_not_create_database_engine_parent(tmp_path) -> None: + archive_path = tmp_path / "offline-parent.jsonl" + written = write_archive( + archive_path, + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260828_0005", + has_more_eligible=False, + ) + database_parent = tmp_path / "must-not-be-created" + database_path = database_parent / "offline.db" + backend_root = Path(__file__).resolve().parents[1] + environment = os.environ.copy() + environment.pop("DATABASE_URL", None) + environment["LLMBENCHLAB_DATABASE_URL"] = f"sqlite:///{database_path}" + + completed = subprocess.run( + [ + sys.executable, + "-m", + "app.cli.audit_retention", + "verify", + "--archive", + str(archive_path), + "--expected-sha256", + written.archive_sha256, + ], + cwd=backend_root, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + + assert completed.returncode == 0 + assert completed.stderr == "" + assert not database_parent.exists() + + +def test_parser_failure_is_fixed_and_never_reflects_unknown_values(capsys) -> None: + marker = "/private/archive-path-marker.jsonl" + + status = audit_retention.main(["verify", "--archive", "/safe", "--unexpected", marker]) + captured = capsys.readouterr() + + assert status == 2 + assert captured.out == "" + assert captured.err == ( + "status=failed operation=parse error_code=audit_retention_argument_invalid " + "error_type=AuditRetentionArgumentError\n" + ) + assert marker not in captured.err + + +def test_verify_rejects_private_fifo_without_blocking_or_reflecting_path(tmp_path) -> None: + marker = "secret-fifo-path-marker" + fifo_path = tmp_path / marker + os.mkfifo(fifo_path, mode=0o600) + backend_root = Path(__file__).resolve().parents[1] + + completed = subprocess.run( + [ + sys.executable, + "-m", + "app.cli.audit_retention", + "verify", + "--archive", + str(fifo_path), + ], + cwd=backend_root, + check=False, + capture_output=True, + text=True, + timeout=5, + ) + + assert completed.returncode == 2 + assert "error_code=archive_input_not_regular" in completed.stderr + assert marker not in completed.stdout + completed.stderr diff --git a/backend/tests/test_evaluation_runner_reliability.py b/backend/tests/test_evaluation_runner_reliability.py index 9a3abe3..a7df640 100644 --- a/backend/tests/test_evaluation_runner_reliability.py +++ b/backend/tests/test_evaluation_runner_reliability.py @@ -32,6 +32,7 @@ ResponseDisposition, RunLease, ) +from app.worker_progress import WorkerProgressEvent def _assert_outside_event_loop() -> None: @@ -128,6 +129,14 @@ async def aclose(self) -> None: self.close_calls += 1 +class _ProgressObserver: + def __init__(self) -> None: + self.events: list[WorkerProgressEvent] = [] + + def note(self, event: WorkerProgressEvent) -> None: + self.events.append(event) + + class _ControlledRunner(EvaluationRunner): def __init__( self, @@ -523,6 +532,27 @@ async def test_cancellation_stops_consumers_before_the_next_question( assert runner.repository.failed == [] +@pytest.mark.asyncio +async def test_runner_notes_only_committed_claim_response_and_terminal_progress( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "app.runners.evaluation_runner.build_adapter", lambda *args, **kwargs: object() + ) + runner = _CancellationRunner(concurrency=1, question_count=1) + observer = _ProgressObserver() + runner._progress_observer = observer + runner.block.set() + + assert await runner.execute("run-controlled") is True + + assert observer.events == [ + WorkerProgressEvent.CLAIM, + WorkerProgressEvent.PROGRESS, + WorkerProgressEvent.PROGRESS, + ] + + @pytest.mark.asyncio async def test_finish_log_uses_terminal_state_resolved_inside_database_lock( monkeypatch: pytest.MonkeyPatch, @@ -610,6 +640,8 @@ async def test_blocking_heartbeat_repository_call_does_not_block_event_loop() -> runner = object.__new__(EvaluationRunner) runner._lease_repository = repository runner._heartbeat_seconds = 0.001 + observer = _ProgressObserver() + runner._progress_observer = observer lease = _FakeLeaseRepository().lease stop = asyncio.Event() lease_lost = asyncio.Event() @@ -627,6 +659,7 @@ async def release_from_event_loop() -> None: assert repository.entered.is_set() assert not lease_lost.is_set() + assert observer.events == [WorkerProgressEvent.LEASE_HEARTBEAT] @pytest.mark.asyncio diff --git a/backend/tests/test_logging.py b/backend/tests/test_logging.py index 4983ef2..e10244d 100644 --- a/backend/tests/test_logging.py +++ b/backend/tests/test_logging.py @@ -5,48 +5,277 @@ import io import json import logging +import math -from app.core.logging import SanitizedJsonFormatter, correlation_scope, request_scope +from app.core.logging import ( + SanitizedJsonFormatter, + configure_logging, + correlation_scope, + request_scope, +) + +REQUEST_ID = "00000000-0000-4000-8000-000000000001" +RUN_ID = "00000000-0000-4000-8000-000000000002" +QUESTION_ID = "00000000-0000-4000-8000-000000000003" +WORKER_GENERATION_ID = "00000000-0000-4000-8000-000000000004" def test_json_formatter_serializes_allowlisted_context_and_redacts_exception_text() -> None: output = io.StringIO() handler = logging.StreamHandler(output) handler.setFormatter(SanitizedJsonFormatter()) - logger = logging.getLogger("llmbenchlab-test-json") - logger.handlers = [handler] - logger.propagate = False - logger.setLevel(logging.INFO) + logger = logging.getLogger("app.runners.evaluation_runner") + previous_handlers = list(logger.handlers) + previous_propagate = logger.propagate + previous_level = logger.level + previous_disabled = logger.disabled secret = "postgresql://user:password@database/private-provider-body" - with request_scope("request-1"), correlation_scope("run-1"): - try: - raise RuntimeError(secret) - except RuntimeError: - logger.exception( - "Sanitized operation failed", - extra={ - "event": "operation_failed", - "run_id": "run-1", - "question_id": "question-1", - "worker_id": "worker-1", - "attempt": 2, - "lease_token": 3, - "untrusted_detail": secret, - }, - ) + try: + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + logger.disabled = False + with request_scope(REQUEST_ID), correlation_scope(RUN_ID): + try: + raise RuntimeError(secret) + except RuntimeError: + logger.exception( + "Question processing failed", + extra={ + "event": "question_processing_failed", + "run_id": RUN_ID, + "question_id": QUESTION_ID, + "worker_id": f"worker:test-host:42:{WORKER_GENERATION_ID}", + "attempt": 2, + "lease_token": 3, + "untrusted_detail": secret, + }, + ) + finally: + logger.handlers = previous_handlers + logger.propagate = previous_propagate + logger.setLevel(previous_level) + logger.disabled = previous_disabled line = output.getvalue().strip() payload = json.loads(line) - assert payload["message"] == "Sanitized operation failed" - assert payload["event"] == "operation_failed" - assert payload["request_id"] == "request-1" - assert payload["correlation_id"] == "run-1" - assert payload["run_id"] == "run-1" - assert payload["question_id"] == "question-1" - assert payload["worker_id"] == "worker-1" + assert payload["message"] == "Question processing failed" + assert payload["event"] == "question_processing_failed" + assert payload["request_id"] == REQUEST_ID + assert payload["correlation_id"] == RUN_ID + assert payload["run_id"] == RUN_ID + assert payload["question_id"] == QUESTION_ID + assert payload["worker_id"] == f"worker:{WORKER_GENERATION_ID}" assert payload["attempt"] == 2 assert payload["lease_token"] == 3 - assert payload["exception_type"] == "RuntimeError" + assert payload["exception_type"] == "suppressed" assert secret not in line assert "untrusted_detail" not in payload + + +def test_external_logger_message_and_identity_are_not_serialized() -> None: + secret = "https://user:password@example.invalid/private?api_key=secret" + record = logging.LogRecord( + name=f"httpx.{secret}", + level=logging.ERROR, + pathname=__file__, + lineno=1, + msg=f"HTTP request failed at {secret}", + args=(), + exc_info=None, + ) + record.request_id = secret + record.correlation_id = secret + record.request_path = secret + record.error_code = secret + record.run_id = secret + record.levelname = secret + record.created = math.inf + + rendered = SanitizedJsonFormatter().format(record) + payload = json.loads(rendered) + + assert payload["logger"] == "external" + assert payload["message"] == "External component log event" + assert payload["level"] == "UNKNOWN" + assert payload["timestamp"] == "1970-01-01T00:00:00+00:00" + assert set(payload) == {"timestamp", "level", "logger", "message"} + assert secret not in rendered + + +def test_unregistered_app_prefix_cannot_spoof_application_logger() -> None: + secret = "sk-app-prefix-spoof-must-not-be-reflected" + record = logging.LogRecord( + name=f"app.{secret}", + level=logging.ERROR, + pathname=__file__, + lineno=1, + msg=secret, + args=(), + exc_info=None, + ) + record.event = secret + + rendered = SanitizedJsonFormatter().format(record) + + assert json.loads(rendered)["logger"] == "external" + assert secret not in rendered + + +def test_application_structured_fields_reject_attacker_selected_strings() -> None: + secret = "sk-structured-canary-must-not-be-reflected" + record = logging.LogRecord( + name="app.workers.service", + level=logging.WARNING, + pathname=__file__, + lineno=1, + msg="Discarding invalid Run queue notification", + args=(), + exc_info=None, + ) + record.request_id = secret + record.correlation_id = secret + record.event = secret + record.run_id = secret + record.question_id = secret + record.worker_id = secret + record.message_id = secret + record.request_method = secret + record.request_path = f"/{secret}" + record.result = secret + record.error_code = secret + record.component = secret + record.attempt = secret + record.lease_token = secret + record.status_code = secret + record.duration_ms = secret + + rendered = SanitizedJsonFormatter().format(record) + payload = json.loads(rendered) + + assert payload == { + "timestamp": payload["timestamp"], + "level": "WARNING", + "logger": "app.workers.service", + "message": "Discarding invalid Run queue notification", + "event": "unsupported", + "request_method": "unsupported", + "request_path": "", + "result": "unsupported", + "error_code": "unsupported", + "component": "unsupported", + } + assert secret not in rendered + + +def test_known_structured_contract_normalizes_exception_family_and_stream_id() -> None: + record = logging.LogRecord( + name="app.main", + level=logging.ERROR, + pathname=__file__, + lineno=1, + msg="API request failed", + args=(), + exc_info=None, + ) + record.event = "api_request_failed" + record.request_id = REQUEST_ID + record.request_method = "GET" + record.request_path = "/live" + record.message_id = "123-4" + record.status_code = 500 + record.duration_ms = 1.25 + record.error_code = "api_error:RuntimeError" + record.result = "internal_server_error" + + payload = json.loads(SanitizedJsonFormatter().format(record)) + + assert payload["request_id"] == REQUEST_ID + assert payload["request_method"] == "GET" + assert payload["request_path"] == "/live" + assert payload["message_id"] == "123-4" + assert payload["status_code"] == 500 + assert payload["duration_ms"] == 1.25 + assert payload["error_code"] == "api_error" + assert payload["result"] == "internal_server_error" + + +def test_non_finite_structured_value_is_omitted_and_never_emits_invalid_json() -> None: + record = logging.LogRecord( + name="app.main", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="Finite structured logging boundary", + args=(), + exc_info=None, + ) + record.duration_ms = math.inf + + rendered = SanitizedJsonFormatter().format(record) + + assert "duration_ms" not in json.loads(rendered) + assert "Infinity" not in rendered + + +def test_huge_integer_timestamp_and_duration_fail_closed_without_formatter_error() -> None: + record = logging.LogRecord( + name="app.main", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="API request completed", + args=(), + exc_info=None, + ) + record.created = 10**10_000 + record.duration_ms = 10**10_000 + + payload = json.loads(SanitizedJsonFormatter().format(record)) + + assert payload["timestamp"] == "1970-01-01T00:00:00+00:00" + assert "duration_ms" not in payload + + +def test_configure_logging_disables_raw_uvicorn_access_handler() -> None: + root_logger = logging.getLogger() + previous_root_handlers = list(root_logger.handlers) + previous_root_level = root_logger.level + external_logger_names = ( + "httpcore", + "httpx", + "redis", + "sqlalchemy", + "sqlalchemy.engine", + "uvicorn", + "uvicorn.error", + ) + previous_external = { + name: (list(logging.getLogger(name).handlers), logging.getLogger(name).propagate) + for name in external_logger_names + } + access_logger = logging.getLogger("uvicorn.access") + previous_disabled = access_logger.disabled + previous_handlers = list(access_logger.handlers) + previous_propagate = access_logger.propagate + try: + access_logger.disabled = False + access_logger.handlers = [logging.StreamHandler(io.StringIO())] + access_logger.propagate = True + + configure_logging("INFO") + + assert access_logger.disabled is True + assert access_logger.handlers == [] + assert access_logger.propagate is False + finally: + root_logger.handlers = previous_root_handlers + root_logger.setLevel(previous_root_level) + for name, (handlers, propagate) in previous_external.items(): + external_logger = logging.getLogger(name) + external_logger.handlers = handlers + external_logger.propagate = propagate + access_logger.disabled = previous_disabled + access_logger.handlers = previous_handlers + access_logger.propagate = previous_propagate diff --git a/backend/tests/test_logging_sources.py b/backend/tests/test_logging_sources.py new file mode 100644 index 0000000..5b48612 --- /dev/null +++ b/backend/tests/test_logging_sources.py @@ -0,0 +1,134 @@ +"""Production log sources keep dynamic and exception values out of messages.""" + +from __future__ import annotations + +import ast +import asyncio +import json +import logging +import runpy +import sys +from pathlib import Path + +import pytest + +from app.core.logging import SanitizedJsonFormatter + +APP_ROOT = Path(__file__).resolve().parents[1] / "app" +LOG_METHODS = frozenset({"debug", "info", "warning", "error", "critical", "exception"}) + + +def _logging_calls(path: Path) -> list[ast.Call]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + calls: list[ast.Call] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute): + continue + if node.func.attr not in LOG_METHODS: + continue + if isinstance(node.func.value, ast.Name) and node.func.value.id == "logger": + calls.append(node) + return calls + + +def test_all_production_log_messages_are_literal_and_argument_free() -> None: + violations: list[str] = [] + for path in sorted(APP_ROOT.rglob("*.py")): + for call in _logging_calls(path): + line = getattr(call, "lineno", 0) + if ( + not call.args + or not isinstance(call.args[0], ast.Constant) + or not isinstance(call.args[0].value, str) + ): + violations.append(f"{path.relative_to(APP_ROOT)}:{line}:non_literal_message") + if len(call.args) != 1: + violations.append(f"{path.relative_to(APP_ROOT)}:{line}:format_arguments") + if isinstance(call.func, ast.Attribute) and call.func.attr == "exception": + violations.append(f"{path.relative_to(APP_ROOT)}:{line}:logger_exception") + if any(keyword.arg == "exc_info" for keyword in call.keywords): + violations.append(f"{path.relative_to(APP_ROOT)}:{line}:exc_info") + + assert violations == [] + + +def test_all_production_loggers_and_literal_structured_values_are_registered() -> None: + violations: list[str] = [] + formatter = SanitizedJsonFormatter() + for path in sorted(APP_ROOT.rglob("*.py")): + calls = _logging_calls(path) + if not calls: + continue + module_name = "app." + ".".join(path.relative_to(APP_ROOT).with_suffix("").parts) + record = logging.LogRecord( + name=module_name, + level=logging.INFO, + pathname=str(path), + lineno=1, + msg="Registered application log source", + args=(), + exc_info=None, + ) + payload = json.loads(formatter.format(record)) + if payload["logger"] != module_name: + violations.append(f"{path.relative_to(APP_ROOT)}:unregistered_logger") + + for call in calls: + message = call.args[0] + assert isinstance(message, ast.Constant) and isinstance(message.value, str) + record.msg = message.value + if json.loads(formatter.format(record))["message"] != message.value: + violations.append( + f"{path.relative_to(APP_ROOT)}:{call.lineno}:unregistered_message" + ) + for keyword in call.keywords: + if keyword.arg != "extra" or not isinstance(keyword.value, ast.Dict): + continue + for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): + if ( + not isinstance(key, ast.Constant) + or key.value not in {"event", "result", "error_code"} + or not isinstance(value, ast.Constant) + or not isinstance(value.value, str) + ): + continue + setattr(record, key.value, value.value) + rendered = json.loads(formatter.format(record)) + if rendered.get(key.value) != value.value: + violations.append( + f"{path.relative_to(APP_ROOT)}:{call.lineno}:" + f"unregistered_{key.value}:{value.value}" + ) + delattr(record, key.value) + + assert violations == [] + + +def test_worker_module_entrypoint_keeps_registered_logger_without_running_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Exercise the real ``python -m`` name without starting I/O or the event loop.""" + + requested_logger_names: list[str | None] = [] + original_get_logger = logging.getLogger + + def capture_logger(name: str | None = None) -> logging.Logger: + requested_logger_names.append(name) + return original_get_logger(name) + + def close_worker_coroutine(coroutine) -> int: + coroutine.close() + return 0 + + # A previously imported module would prevent runpy from reproducing the + # ``__main__`` execution contract. MonkeyPatch restores it after the test. + monkeypatch.delitem(sys.modules, "app.worker", raising=False) + monkeypatch.setattr(logging, "getLogger", capture_logger) + monkeypatch.setattr(asyncio, "run", close_worker_coroutine) + + with pytest.raises(SystemExit) as exit_info: + runpy.run_module("app.worker", run_name="__main__") + + assert exit_info.value.code == 0 + assert "app.worker" in requested_logger_names + assert "__main__" not in requested_logger_names diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py index b29d6c1..011e3cd 100644 --- a/backend/tests/test_migrations.py +++ b/backend/tests/test_migrations.py @@ -13,6 +13,7 @@ from app.db.base import Base from app.db.prepare_migrations import ( CREDENTIAL_REVISION, + GOVERNANCE_REVISION, LEGACY_REVISION, PHASE_1_REVISION, SchemaPreparationError, @@ -24,7 +25,7 @@ from app.db.session import create_database_engine BACKEND_ROOT = Path(__file__).resolve().parents[1] -HEAD_REVISION = "20260827_0004" +HEAD_REVISION = "20260828_0005" WEB_CREDENTIAL_REVISION = CREDENTIAL_REVISION RELIABILITY_REVISION = "20260825_0002" @@ -386,6 +387,40 @@ def test_prepare_backs_up_and_adopts_web_credential_schema( _run_alembic(database_path, "check") +@pytest.mark.parametrize("versioned", [False, True]) +def test_prepare_backs_up_and_adopts_governance_schema( + tmp_path: Path, + versioned: bool, +) -> None: + database_path = tmp_path / f"governance-{'versioned' if versioned else 'unversioned'}.db" + _run_alembic(database_path, "upgrade", GOVERNANCE_REVISION) + if not versioned: + with sqlite3.connect(database_path) as connection: + connection.execute("DELETE FROM alembic_version") + + result = prepare_database(_database_url(database_path)) + + assert result.action == ("versioned_governance" if versioned else "stamped_governance") + assert result.stamped_revision == GOVERNANCE_REVISION + assert result.backup_path is not None and result.backup_path.is_file() + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) + with sqlite3.connect(result.backup_path) as backup: + tables = { + row[0] for row in backup.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + indexes = { + row[0] for row in backup.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert "audit_events" in tables + assert "worker_processes" not in tables + assert "ix_audit_events_expires_id" not in indexes + assert "ix_audit_events_occurred_id" not in indexes + + _run_alembic(database_path, "upgrade", "head") + assert _read_heads(database_path) == (HEAD_REVISION,) + _run_alembic(database_path, "check") + + @pytest.mark.parametrize( ("cancellation_requested", "expected_status", "expected_last_error"), [ @@ -550,6 +585,7 @@ def test_governance_schema_matches_orm_and_does_not_seed_policy(tmp_path: Path) "question_executions", "provider_call_reservations", "audit_events", + "worker_processes", } with sqlite3.connect(database_path) as connection: tables = { @@ -557,6 +593,52 @@ def test_governance_schema_matches_orm_and_does_not_seed_policy(tmp_path: Path) for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'") } assert expected_tables <= tables + worker_columns = { + row[1] for row in connection.execute("PRAGMA table_info(worker_processes)") + } + assert worker_columns == { + "generation_id", + "worker_id", + "started_at", + "last_seen_at", + "last_scan_at", + "last_claim_at", + "last_progress_at", + "last_lease_heartbeat_at", + "stopped_at", + } + worker_indexes = { + row[1]: row for row in connection.execute("PRAGMA index_list(worker_processes)") + } + assert worker_indexes["ix_worker_processes_stopped_seen_generation"][2] == 0 + assert [ + row[2] + for row in connection.execute( + "PRAGMA index_info(ix_worker_processes_stopped_seen_generation)" + ) + ] == ["stopped_at", "last_seen_at", "generation_id"] + audit_indexes = { + row[1]: row for row in connection.execute("PRAGMA index_list(audit_events)") + } + for index_name, expected_columns in ( + ("ix_audit_events_expires_id", ["expires_at", "id"]), + ("ix_audit_events_occurred_id", ["occurred_at", "id"]), + ): + assert audit_indexes[index_name][2] == 0 + assert [ + row[2] for row in connection.execute(f'PRAGMA index_info("{index_name}")') + ] == expected_columns + audit_window_plan = [ + row[3] + for row in connection.execute( + "EXPLAIN QUERY PLAN SELECT * FROM audit_events " + "WHERE occurred_at >= ? AND occurred_at < ? " + "ORDER BY occurred_at, id LIMIT 50001", + ("2026-08-28 00:00:00", "2026-08-28 00:15:00"), + ) + ] + assert any("ix_audit_events_occurred_id" in detail for detail in audit_window_plan) + assert all("TEMP B-TREE" not in detail for detail in audit_window_plan) assert connection.execute("SELECT COUNT(*) FROM governance_policies").fetchone()[0] == 0 policy_indexes = { row[1]: row for row in connection.execute("PRAGMA index_list(governance_policies)") @@ -705,6 +787,28 @@ def test_governance_upgrade_backfills_failed_attempts_by_run_state( ) +def test_worker_progress_downgrade_rejects_process_facts_before_ddl(tmp_path: Path) -> None: + database_path = tmp_path / "worker-progress-downgrade.db" + _run_alembic(database_path, "upgrade", "head") + with sqlite3.connect(database_path) as connection: + connection.execute( + "INSERT INTO worker_processes (generation_id, worker_id, started_at, last_seen_at) " + "VALUES ('00000000-0000-0000-0000-000000000001', 'worker-test', " + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + ) + + failed = _invoke_alembic(database_path, "downgrade", GOVERNANCE_REVISION) + + assert failed.returncode != 0 + assert "process facts exist" in failed.stderr + assert _read_heads(database_path) == (HEAD_REVISION,) + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT COUNT(*) FROM worker_processes").fetchone()[0] == 1 + audit_indexes = {row[1] for row in connection.execute("PRAGMA index_list(audit_events)")} + assert "ix_audit_events_expires_id" in audit_indexes + assert "ix_audit_events_occurred_id" in audit_indexes + + def test_governance_downgrade_rejects_response_metadata_before_ddl(tmp_path: Path) -> None: database_path = tmp_path / "governance-metadata-downgrade.db" _run_alembic(database_path, "upgrade", LEGACY_REVISION) @@ -720,7 +824,7 @@ def test_governance_downgrade_rejects_response_metadata_before_ddl(tmp_path: Pat assert failed.returncode != 0 assert "Response Provider metadata exists" in failed.stderr - assert _read_heads(database_path) == (HEAD_REVISION,) + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) with sqlite3.connect(database_path) as connection: tables = { row[0] @@ -753,7 +857,7 @@ def test_governance_downgrade_rejects_failure_and_fairness_evidence_before_ddl( assert failed.returncode != 0 assert "Run governance/failure evidence" in failed.stderr - assert _read_heads(database_path) == (HEAD_REVISION,) + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) with sqlite3.connect(database_path) as connection: assert connection.execute( "SELECT failed_attempt_count, dispatch_count FROM evaluation_runs WHERE id = 'run-1'" @@ -781,7 +885,7 @@ def test_governance_downgrade_rejects_question_execution_before_ddl(tmp_path: Pa assert failed.returncode != 0 assert "question-execution evidence exists" in failed.stderr - assert _read_heads(database_path) == (HEAD_REVISION,) + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) with sqlite3.connect(database_path) as connection: assert connection.execute( "SELECT run_id, question_id FROM question_executions WHERE id = 'execution-1'" @@ -870,7 +974,7 @@ def test_governance_ledger_foreign_keys_are_never_delete_restrict(tmp_path: Path assert failed.returncode != 0 assert "ledger, audit, policy, scope" in failed.stderr - assert _read_heads(database_path) == (HEAD_REVISION,) + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) def test_reliability_downgrade_rejects_active_runs_before_ddl(tmp_path: Path) -> None: diff --git a/backend/tests/test_phase2_acceptance_script.py b/backend/tests/test_phase2_acceptance_script.py index 9cd73b1..7df2d19 100644 --- a/backend/tests/test_phase2_acceptance_script.py +++ b/backend/tests/test_phase2_acceptance_script.py @@ -28,6 +28,181 @@ def _load_script() -> ModuleType: script = _load_script() +def test_migration_acceptance_retains_both_populated_guards_and_empty_round_trips() -> None: + source = _ACCEPTANCE_SCRIPT.read_text(encoding="utf-8") + + assert script.PRE_GOVERNANCE_REVISION == "20260827_0003" + assert script.GOVERNANCE_REVISION == "20260827_0004" + assert script.WORKER_PROGRESS_REVISION == "20260828_0005" + assert "postgres_populated_0005_and_0004_downgrade_guards_with_empty_round_trips" in source + assert "Cannot downgrade Worker progress schema" in source + assert "Cannot downgrade governance schema" in source + assert "application_worker_progress_0005_to_0004_guard" in source + assert "isolated_governance_0004_to_0003_guard" in source + assert "worker_facts_cleared_in_clone_only" in source + assert 'TEMPLATE "llmbenchlab"' in source + assert "worker_progress_0005_to_0004_round_trip" in source + assert "governance_0004_to_0003_round_trip" in source + assert "pre_worker_progress_revision" in source + assert "pre_governance_revision" in source + + +def test_acceptance_declares_the_two_worker_expected_minimum() -> None: + source = _ACCEPTANCE_SCRIPT.read_text(encoding="utf-8") + + assert '"LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES": "2"' in source + + +def test_worker_progress_health_requires_both_main_loops() -> None: + harness = _bare_harness() + responses = [ + { + "status_code": 200, + "payload": { + "worker_expected_processes": 2, + "worker_registered_processes": 2, + "worker_live_processes": 1, + "worker_stalled_processes": 1, + "worker_shortfall_processes": 1, + }, + }, + { + "status_code": 200, + "payload": { + "worker_expected_processes": 2, + "worker_registered_processes": 2, + "worker_live_processes": 2, + "worker_stalled_processes": 0, + "worker_shortfall_processes": 0, + }, + }, + ] + requests: list[tuple[object, ...]] = [] + + def fake_http_json(*args: object, **kwargs: object) -> dict[str, object]: + requests.append((*args, kwargs)) + return responses.pop(0) + + def fake_wait_for( + description: str, + probe: object, + predicate: object, + *, + timeout: float, + interval: float, + ) -> dict[str, object]: + assert ( + description == "two registered and live Worker main loops without stalls or shortfall" + ) + assert timeout == 12 + assert interval == 0.25 + assert callable(probe) + assert callable(predicate) + degraded = probe() + assert predicate(degraded) is False + healthy = probe() + assert predicate(healthy) is True + return healthy + + harness.http_json = fake_http_json + harness.wait_for = fake_wait_for + + result = harness.wait_worker_progress_healthy(timeout=12) + + assert result == { + "worker_expected_processes": 2, + "worker_registered_processes": 2, + "worker_live_processes": 2, + "worker_stalled_processes": 0, + "worker_shortfall_processes": 0, + } + assert requests == [ + ("GET", "/tasks/metrics", {"accepted": {200}, "timeout": 4}), + ("GET", "/tasks/metrics", {"accepted": {200}, "timeout": 4}), + ] + + +def test_topology_records_database_derived_worker_progress() -> None: + source = _ACCEPTANCE_SCRIPT.read_text(encoding="utf-8") + + assert "worker_progress = self.wait_worker_progress_healthy()" in source + assert '"worker_progress": worker_progress' in source + + +def test_template_clone_blocks_new_source_connections_and_restores_access() -> None: + harness = _bare_harness() + calls: list[tuple[str, str, bool]] = [] + + def fake_psql_database( + sql: str, + *, + database: str, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + calls.append((sql, database, check)) + stdout = '{"attempted":2,"all_terminated":true}\n' if "json_build_object" in sql else "" + return subprocess.CompletedProcess(["psql"], 0, stdout=stdout, stderr="") + + harness.psql_database = fake_psql_database + + result = harness.clone_application_database("p2governance_safe") + + assert result == { + "source_connections_disabled": True, + "terminated_connection_count": 2, + "source_connections_restored": True, + "create_returncode": 0, + } + assert calls[0] == ( + 'ALTER DATABASE "llmbenchlab" WITH ALLOW_CONNECTIONS false;', + "postgres", + True, + ) + assert "pg_terminate_backend(pid)" in calls[1][0] + assert "datname = 'llmbenchlab'" in calls[1][0] + assert calls[2] == ( + 'CREATE DATABASE "p2governance_safe" TEMPLATE "llmbenchlab";', + "postgres", + True, + ) + assert calls[3] == ( + 'ALTER DATABASE "llmbenchlab" WITH ALLOW_CONNECTIONS true;', + "postgres", + False, + ) + + +def test_template_clone_restores_source_access_when_create_fails() -> None: + harness = _bare_harness() + calls: list[str] = [] + + def fake_psql_database( + sql: str, + *, + database: str, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + assert database == "postgres" + calls.append(sql) + if "json_build_object" in sql: + return subprocess.CompletedProcess( + ["psql"], + 0, + stdout='{"attempted":0,"all_terminated":true}\n', + stderr="", + ) + if sql.startswith("CREATE DATABASE"): + raise script.AcceptanceFailure("simulated clone failure") + return subprocess.CompletedProcess(["psql"], 0, stdout="", stderr="") + + harness.psql_database = fake_psql_database + + with pytest.raises(script.AcceptanceFailure, match="simulated clone failure"): + harness.clone_application_database("p2governance_safe") + + assert calls[-1] == 'ALTER DATABASE "llmbenchlab" WITH ALLOW_CONNECTIONS true;' + + def _bare_harness() -> object: harness = object.__new__(script.Phase2Acceptance) harness.require = script.Phase2Acceptance.require.__get__(harness) diff --git a/backend/tests/test_phase2_capacity_script.py b/backend/tests/test_phase2_capacity_script.py index 2bdccbb..d772199 100644 --- a/backend/tests/test_phase2_capacity_script.py +++ b/backend/tests/test_phase2_capacity_script.py @@ -330,6 +330,7 @@ def test_formal_profile_is_one_fixed_switch_with_an_independent_schema() -> None harness = script.Phase2Capacity(_REPOSITORY_ROOT, args) try: assert harness.formal_slo_v2 is True + assert harness.env["LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES"] == "2" assert harness.evidence["schema_version"] == script.CAPACITY_EVIDENCE_SCHEMA_V2 assert ( harness.evidence["configuration"]["qualification_profile"] @@ -399,6 +400,7 @@ def test_default_profile_retains_v1_schema_and_single_burst_identity() -> None: harness = script.Phase2Capacity(_REPOSITORY_ROOT, script.parse_arguments([])) try: assert harness.formal_slo_v2 is False + assert harness.env["LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES"] == "2" assert harness.evidence["schema_version"] == script.CAPACITY_EVIDENCE_SCHEMA_V1 assert ( harness.evidence["configuration"]["qualification_profile"] @@ -452,6 +454,11 @@ def test_default_profile_retains_v1_schema_and_single_burst_identity() -> None: assert configurable.run_concurrency == 2 assert configurable.lease_seconds == 32 assert configurable.redis_operation_timeout_seconds == 1.1 + configurable_harness = script.Phase2Capacity(_REPOSITORY_ROOT, configurable) + try: + assert configurable_harness.env["LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES"] == "3" + finally: + configurable_harness._credential_secret_dir.cleanup() def test_demo_producer_retains_slo_identity_fields() -> None: @@ -1957,14 +1964,25 @@ def test_capacity_harness_is_mock_only_sanitized_and_make_addressable() -> None: } -def test_acceptance_requires_populated_0004_refusal_and_empty_round_trip() -> None: +def test_acceptance_requires_both_populated_guards_and_empty_round_trips() -> None: source = _ACCEPTANCE_SCRIPT.read_text(encoding="utf-8") - assert "postgres_populated_0004_downgrade_refusal_and_empty_round_trip" in source + assert "postgres_populated_0005_and_0004_downgrade_guards_with_empty_round_trips" in source + assert "Cannot downgrade Worker progress schema" in source assert "Cannot downgrade governance schema" in source + assert "worker_processes" in source + assert 'PRE_GOVERNANCE_REVISION = "20260827_0003"' in source + assert 'GOVERNANCE_REVISION = "20260827_0004"' in source + assert 'WORKER_PROGRESS_REVISION = "20260828_0005"' in source + assert "application_worker_progress_0005_to_0004_guard" in source + assert "isolated_governance_0004_to_0003_guard" in source + assert "worker_progress_0005_to_0004_round_trip" in source + assert "governance_0004_to_0003_round_trip" in source assert "p2roundtrip_" in source + assert "p2governance_" in source assert "DATABASE_URL=" in source and "empty_database_url" in source - assert 'DROP DATABASE IF EXISTS "{}" WITH (FORCE)' in source + assert source.count("DROP DATABASE IF EXISTS") >= 2 + assert source.count("WITH (FORCE)") >= 2 def test_acceptance_command_failure_redacts_database_credentials() -> None: diff --git a/backend/tests/test_prometheus_alert_rules.py b/backend/tests/test_prometheus_alert_rules.py new file mode 100644 index 0000000..2d4f7c8 --- /dev/null +++ b/backend/tests/test_prometheus_alert_rules.py @@ -0,0 +1,137 @@ +"""Static contract checks for the repository-owned Phase 2 alert rules.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[2] +_RULES = _ROOT / "deploy" / "observability" / "prometheus-alerts.json" +_SCRAPE = _ROOT / "deploy" / "observability" / "prometheus-scrape.example.yml" +_OPERATIONS = _ROOT / "docs" / "OPERATIONS.md" + +_EXPECTED_RULES = { + "LLMBenchLabMetricsUnavailable": ( + 'up{job="llmbenchlab"} == 0', + "2m", + "critical", + "alert-exporter-unavailable", + ), + "LLMBenchLabBacklogPersistent": ( + "llmbenchlab_runs_due_pending > 0", + "15m", + "warning", + "alert-backlog-persistent", + ), + "LLMBenchLabDeadLettered": ( + 'llmbenchlab_audit_events_window{event_type="run_dead_lettered"} > 0', + "0s", + "critical", + "alert-dead-letter", + ), + "LLMBenchLabGovernanceIntegrityError": ( + 'llmbenchlab_audit_events_window{event_type="governance_integrity_error"} > 0', + "0s", + "critical", + "alert-governance-integrity", + ), + "LLMBenchLabGovernanceOverdrawn": ( + "llmbenchlab_governance_scopes_overdrawn > 0", + "0s", + "critical", + "alert-governance-overdraw", + ), + "LLMBenchLabQueueDegraded": ( + "llmbenchlab_queue_configured == 1 and llmbenchlab_queue_available == 0", + "2m", + "warning", + "alert-queue-degraded", + ), + "LLMBenchLabWorkerStalled": ( + "llmbenchlab_worker_shortfall > 0", + "2m", + "critical", + "alert-worker-stalled", + ), + "LLMBenchLabLeaseRecoverySlow": ( + "llmbenchlab_run_expired_lease_oldest_age_seconds > " + "llmbenchlab_run_recovery_alert_threshold_seconds", + "1m", + "warning", + "alert-lease-recovery-slow", + ), +} + +_DELIVERED_METRICS = { + "up", + "llmbenchlab_runs_due_pending", + "llmbenchlab_audit_events_window", + "llmbenchlab_governance_scopes_overdrawn", + "llmbenchlab_queue_configured", + "llmbenchlab_queue_available", + "llmbenchlab_worker_shortfall", + "llmbenchlab_run_expired_lease_oldest_age_seconds", + "llmbenchlab_run_recovery_alert_threshold_seconds", +} + + +def test_prometheus_alert_rules_are_the_exact_low_cardinality_contract() -> None: + document = json.loads(_RULES.read_text(encoding="utf-8")) + + assert set(document) == {"groups"} + assert len(document["groups"]) == 1 + group = document["groups"][0] + assert set(group) == {"name", "interval", "rules"} + assert group["name"] == "llmbenchlab.phase2" + assert group["interval"] == "30s" + assert len(group["rules"]) == 8 + assert {rule["alert"] for rule in group["rules"]} == set(_EXPECTED_RULES) + + for rule in group["rules"]: + assert set(rule) == {"alert", "expr", "for", "labels", "annotations"} + expression, duration, severity, anchor = _EXPECTED_RULES[rule["alert"]] + assert rule["expr"] == expression + assert rule["for"] == duration + assert rule["labels"] == { + "severity": severity, + "component": "llmbenchlab-control-plane", + } + assert set(rule["annotations"]) == { + "summary", + "description", + "runbook_url", + "silence_policy", + } + assert rule["annotations"]["summary"] + assert rule["annotations"]["description"] + assert "owner" in rule["annotations"]["silence_policy"].lower() or rule["alert"] in { + "LLMBenchLabGovernanceIntegrityError", + "LLMBenchLabGovernanceOverdrawn", + } + assert rule["annotations"]["runbook_url"] == ( + "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/" + f"docs/OPERATIONS.md#{anchor}" + ) + assert "rate(" not in expression + assert "increase(" not in expression + + identifiers = set(re.findall(r"\b(?:up|llmbenchlab_[a-z0-9_]+)\b", expression)) + assert identifiers <= _DELIVERED_METRICS + + +def test_prometheus_scrape_example_uses_the_fixed_endpoint_and_safe_interval() -> None: + text = _SCRAPE.read_text(encoding="utf-8") + + assert "job_name: llmbenchlab" in text + assert "scrape_interval: 60s" in text + assert "metrics_path: /api/v1/metrics/prometheus" in text + assert "api:8000" in text + assert "30s" not in text + + +def test_prometheus_alert_runbook_targets_exist_exactly_once() -> None: + operations = _OPERATIONS.read_text(encoding="utf-8") + + for _expression, _duration, _severity, anchor in _EXPECTED_RULES.values(): + assert operations.count(f'') == 1 diff --git a/backend/tests/test_prometheus_exporter.py b/backend/tests/test_prometheus_exporter.py new file mode 100644 index 0000000..ada44ae --- /dev/null +++ b/backend/tests/test_prometheus_exporter.py @@ -0,0 +1,660 @@ +"""Bounded, fail-closed Prometheus exporter contract tests.""" + +from __future__ import annotations + +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException +from sqlalchemy import func, select +from sqlalchemy.exc import SQLAlchemyError + +import app.api.v1.observability as observability_api +from app.core.config import get_settings +from app.db.session import SessionLocal +from app.governance.audit import append_audit_event +from app.models import ( + Benchmark, + EvaluationRun, + Model, + ProviderType, + RunStatus, + WorkerProcess, +) +from app.observability import ( + METRICS_AUDIT_EVENT_LIMIT, + TASK_EVENT_TYPES, + MetricsObservationLimitExceeded, + collect_task_history, + configure_read_snapshot, + database_clock, +) +from app.observability.prometheus import PrometheusRenderingError, render_prometheus +from app.task_queue import QueueUnavailable + + +class _UnavailableQueue: + async def ping(self) -> bool: + raise QueueUnavailable("redis://user:queue-secret@queue/private") + + +_EXPECTED_FAMILIES = ( + "llmbenchlab_runs_pending", + "llmbenchlab_runs_due_pending", + "llmbenchlab_runs_running", + "llmbenchlab_runs_expired_running", + "llmbenchlab_runs_cancellation_requested", + "llmbenchlab_runs_retry_scheduled", + "llmbenchlab_runs_dead_lettered", + "llmbenchlab_runs_queue_notification_error", + "llmbenchlab_runs_managed_backlog", + "llmbenchlab_runs_governance_delayed", + "llmbenchlab_runs_governance_exhausted", + "llmbenchlab_provider_attempts_active", + "llmbenchlab_governance_scopes_overdrawn", + "llmbenchlab_run_lease_acquisitions", + "llmbenchlab_run_failed_attempts", + "llmbenchlab_run_dispatches", + "llmbenchlab_run_expired_lease_oldest_age_seconds", + "llmbenchlab_audit_events_window", + "llmbenchlab_audit_event_window_seconds", + "llmbenchlab_metrics_audit_events_scanned", + "llmbenchlab_metrics_audit_event_limit", + "llmbenchlab_run_latency_quantile_seconds", + "llmbenchlab_run_latency_samples", + "llmbenchlab_run_latency_truncated", + "llmbenchlab_run_latency_window_seconds", + "llmbenchlab_metrics_latency_sample_limit", + "llmbenchlab_queue_configured", + "llmbenchlab_queue_available", + "llmbenchlab_worker_processes", + "llmbenchlab_worker_expected_minimum", + "llmbenchlab_worker_shortfall", + "llmbenchlab_worker_activity_observed", + "llmbenchlab_worker_activity_oldest_age_seconds", + "llmbenchlab_worker_stale_threshold_seconds", + "llmbenchlab_run_recovery_alert_threshold_seconds", + "llmbenchlab_metrics_snapshot_unixtime_seconds", +) + + +def _sample_value(body: str, sample: str) -> float: + prefix = sample + " " + line = next(line for line in body.splitlines() if line.startswith(prefix)) + return float(line.removeprefix(prefix)) + + +def _database_now() -> datetime: + with SessionLocal() as session: + value = session.scalar(select(func.current_timestamp())) + assert isinstance(value, datetime) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def test_prometheus_empty_snapshot_has_fixed_text_contract(client) -> None: + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 200 + assert response.headers["content-type"] == "text/plain; version=0.0.4; charset=utf-8" + assert response.headers["cache-control"] == "no-store" + assert "\r" not in response.text + assert response.text.endswith("\n") + assert not response.text.endswith("\n\n") + assert "# TYPE " in response.text + assert " counter\n" not in response.text + assert "_total" not in response.text + assert [ + line.split()[2] for line in response.text.splitlines() if line.startswith("# TYPE ") + ] == list(_EXPECTED_FAMILIES) + assert _sample_value(response.text, "llmbenchlab_runs_pending") == 0 + assert _sample_value(response.text, "llmbenchlab_queue_configured") == 0 + assert _sample_value(response.text, "llmbenchlab_queue_available") == 0 + assert _sample_value(response.text, "llmbenchlab_worker_expected_minimum") == 1 + assert _sample_value(response.text, "llmbenchlab_worker_shortfall") == 1 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="registered"}') == 0 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="live"}') == 0 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="stalled"}') == 0 + for event_type in TASK_EVENT_TYPES: + assert ( + _sample_value( + response.text, + f'llmbenchlab_audit_events_window{{event_type="{event_type}"}}', + ) + == 0 + ) + for activity in ("scan", "claim", "lease_heartbeat", "progress"): + assert ( + _sample_value( + response.text, + f'llmbenchlab_worker_activity_observed{{activity="{activity}"}}', + ) + == 0 + ) + assert ( + f'llmbenchlab_worker_activity_oldest_age_seconds{{activity="{activity}"}}' + not in response.text + ) + assert "llmbenchlab_run_latency_quantile_seconds{" not in response.text + + +def test_prometheus_rejects_dynamic_query_parameters(client) -> None: + response = client.get("/api/v1/metrics/prometheus?window=custom") + + assert response.status_code == 422 + assert response.json() == { + "detail": { + "code": "metrics_query_parameters_not_allowed", + "message": "Metrics scrape does not accept query parameters", + } + } + + +def test_prometheus_validates_window_events_and_exposes_worker_aggregate( + client, + db_session, +) -> None: + now = _database_now() + secret_identity = "worker-secret-identity" + db_session.add( + WorkerProcess( + generation_id="11111111-1111-4111-8111-111111111111", + worker_id=secret_identity, + started_at=now - timedelta(seconds=30), + last_seen_at=now - timedelta(seconds=1), + last_scan_at=now - timedelta(seconds=5), + last_claim_at=now - timedelta(seconds=4), + last_progress_at=now - timedelta(seconds=3), + last_lease_heartbeat_at=now - timedelta(seconds=2), + ) + ) + append_audit_event( + db_session, + event_key="metrics:dead-letter:inside", + event_type="run_dead_lettered", + occurred_at=now - timedelta(seconds=10), + payload={"failed_attempt_count": 3, "reason": "worker_error"}, + correlation_id="metrics-secret-run", + run_id="metrics-secret-run", + model_id="metrics-secret-model", + ) + append_audit_event( + db_session, + event_key="metrics:dead-letter:outside", + event_type="run_dead_lettered", + occurred_at=now - timedelta(seconds=901), + payload={"failed_attempt_count": 3, "reason": "worker_error"}, + correlation_id="metrics-outside-run", + run_id="metrics-outside-run", + ) + db_session.commit() + + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 200 + assert ( + _sample_value( + response.text, + 'llmbenchlab_audit_events_window{event_type="run_dead_lettered"}', + ) + == 1 + ) + assert _sample_value(response.text, "llmbenchlab_metrics_audit_events_scanned") == 1 + assert _sample_value(response.text, "llmbenchlab_metrics_audit_event_limit") == 50_000 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="registered"}') == 1 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="live"}') == 1 + assert _sample_value(response.text, 'llmbenchlab_worker_processes{state="stalled"}') == 0 + assert _sample_value(response.text, "llmbenchlab_worker_shortfall") == 0 + for activity in ("scan", "claim", "lease_heartbeat", "progress"): + assert ( + _sample_value( + response.text, + f'llmbenchlab_worker_activity_observed{{activity="{activity}"}}', + ) + == 1 + ) + assert ( + f'llmbenchlab_worker_activity_oldest_age_seconds{{activity="{activity}"}}' + in response.text + ) + assert secret_identity not in response.text + assert "metrics-secret-run" not in response.text + assert "metrics-secret-model" not in response.text + + +def test_prometheus_worker_oldest_age_and_json_latest_timestamp_are_distinct( + client, + db_session, +) -> None: + now = _database_now() + recent_scan = now - timedelta(seconds=2) + old_scan = now - timedelta(seconds=20) + db_session.add_all( + [ + WorkerProcess( + generation_id="21111111-1111-4111-8111-111111111111", + worker_id="metrics-worker-recent", + started_at=now - timedelta(seconds=40), + last_seen_at=now - timedelta(seconds=1), + last_scan_at=recent_scan, + ), + WorkerProcess( + generation_id="31111111-1111-4111-8111-111111111111", + worker_id="metrics-worker-oldest", + started_at=now - timedelta(seconds=40), + last_seen_at=now - timedelta(seconds=1), + last_scan_at=old_scan, + ), + ] + ) + db_session.commit() + + task_metrics = client.get("/api/v1/tasks/metrics") + prometheus = client.get("/api/v1/metrics/prometheus") + + assert task_metrics.status_code == 200 + assert task_metrics.json()["worker_registered_processes"] == 2 + assert task_metrics.json()["worker_live_processes"] == 2 + assert datetime.fromisoformat(task_metrics.json()["worker_last_scan_at"]) == recent_scan + assert prometheus.status_code == 200 + assert ( + _sample_value( + prometheus.text, + 'llmbenchlab_worker_activity_oldest_age_seconds{activity="scan"}', + ) + >= 20 + ) + + +def test_prometheus_run_latency_uses_the_fixed_one_hour_window(client, db_session) -> None: + now = _database_now() + model = Model( + id="metrics-latency-model", name="Metrics latency", provider_type=ProviderType.MOCK + ) + benchmark = Benchmark( + id="metrics-latency-benchmark", + slug="metrics-latency-benchmark", + name="Metrics latency benchmark", + version="1.0.0", + description="fixture", + dimension="general", + language="en", + license="MIT", + source="local", + evaluator_type="exact_match", + evaluator_config={}, + prompt_template={}, + dataset_hash="metrics-latency-hash", + question_count=1, + ) + db_session.add_all( + [ + model, + benchmark, + EvaluationRun( + id="metrics-latency-inside", + model_id=model.id, + benchmark_id=benchmark.id, + status=RunStatus.COMPLETED, + model_parameters_snapshot={}, + benchmark_hash_snapshot=benchmark.dataset_hash, + prompt_template_snapshot={}, + created_at=now - timedelta(minutes=50), + started_at=now - timedelta(minutes=45), + finished_at=now - timedelta(minutes=40), + ), + EvaluationRun( + id="metrics-latency-outside", + model_id=model.id, + benchmark_id=benchmark.id, + status=RunStatus.COMPLETED, + model_parameters_snapshot={}, + benchmark_hash_snapshot=benchmark.dataset_hash, + prompt_template_snapshot={}, + created_at=now - timedelta(minutes=70), + started_at=now - timedelta(minutes=65), + finished_at=now - timedelta(minutes=61), + ), + ] + ) + db_session.commit() + + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 200 + for phase in ("queue", "execution", "end_to_end"): + assert ( + _sample_value(response.text, f'llmbenchlab_run_latency_samples{{phase="{phase}"}}') == 1 + ) + assert ( + _sample_value(response.text, f'llmbenchlab_run_latency_truncated{{phase="{phase}"}}') + == 0 + ) + assert ( + _sample_value( + response.text, + 'llmbenchlab_run_latency_quantile_seconds{phase="queue",quantile="0.5"}', + ) + == 300 + ) + assert ( + _sample_value( + response.text, + 'llmbenchlab_run_latency_quantile_seconds{phase="execution",quantile="0.5"}', + ) + == 300 + ) + assert ( + _sample_value( + response.text, + 'llmbenchlab_run_latency_quantile_seconds{phase="end_to_end",quantile="0.5"}', + ) + == 600 + ) + + +def test_prometheus_expired_lease_age_is_current_right_censored_fact(client) -> None: + model = client.post( + "/api/v1/models", + json={"name": "Expired lease model", "provider_type": "mock", "enabled": True}, + ) + benchmark = client.post("/api/v1/benchmarks/reload-demo") + run = client.post( + "/api/v1/runs", + json={"model_id": model.json()["id"], "benchmark_id": benchmark.json()["id"]}, + ) + assert run.status_code == 202 + now = _database_now() + with SessionLocal() as session, session.begin(): + persisted = session.get(EvaluationRun, run.json()["id"]) + assert persisted is not None + persisted.status = RunStatus.RUNNING + persisted.lease_owner = "metrics-expired-worker" + persisted.lease_token = 1 + persisted.lease_expires_at = now - timedelta(seconds=5) + persisted.heartbeat_at = now - timedelta(seconds=10) + persisted.started_at = now - timedelta(seconds=30) + + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 200 + assert _sample_value(response.text, "llmbenchlab_runs_expired_running") == 1 + assert ( + _sample_value( + response.text, + "llmbenchlab_run_expired_lease_oldest_age_seconds", + ) + >= 5 + ) + + +def test_prometheus_queue_failure_is_non_authoritative_and_sanitized( + client, + monkeypatch, + caplog, +) -> None: + monkeypatch.setattr(client.app.state, "run_queue", _UnavailableQueue()) + caplog.set_level("WARNING", logger="app.api.v1.observability") + + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 200 + assert _sample_value(response.text, "llmbenchlab_queue_configured") == 1 + assert _sample_value(response.text, "llmbenchlab_queue_available") == 0 + assert "queue-secret" not in response.text + assert "queue-secret" not in caplog.text + assert any(record.event == "metrics_queue_unavailable" for record in caplog.records) + + +def test_prometheus_audit_corruption_fails_whole_scrape_without_reflection( + client, + db_session, + caplog, +) -> None: + marker = "sk-metrics-corruption-never-reflect" + event = append_audit_event( + db_session, + event_key="metrics:corrupt:event", + event_type="run_claimed", + occurred_at=_database_now() - timedelta(seconds=1), + payload={"dispatch_count": 1}, + run_id="metrics-corrupt-run", + worker_id="metrics-corrupt-worker", + attempt=1, + lease_token=1, + ) + event.payload_hash = marker + "x" * (64 - len(marker)) + db_session.commit() + caplog.set_level("ERROR", logger="app.api.v1.observability") + + response = client.get("/api/v1/metrics/prometheus") + + assert response.status_code == 500 + assert response.json() == { + "detail": { + "code": "audit_event_integrity_error", + "message": "A retained audit event failed integrity validation", + } + } + assert marker not in response.text + assert marker not in caplog.text + + +@pytest.mark.parametrize("damage", ["payload_json", "occurred_at"]) +def test_prometheus_and_history_classify_invalid_audit_storage_as_integrity_error( + client, + db_session, + caplog, + damage, +) -> None: + marker = "sk-invalid-audit-json-never-reflect" + occurred_at = _database_now() - timedelta(seconds=1) + event = append_audit_event( + db_session, + event_key="metrics:corrupt:json", + event_type="run_claimed", + occurred_at=occurred_at, + payload={"dispatch_count": 1}, + run_id="metrics-corrupt-json-run", + worker_id="metrics-corrupt-json-worker", + attempt=1, + lease_token=1, + ) + event_id = event.id + db_session.commit() + if damage == "payload_json": + db_session.connection().exec_driver_sql( + "UPDATE audit_events SET payload = ? WHERE id = ?", + ("{" + marker, event_id), + ) + else: + malformed_timestamp = occurred_at.replace(tzinfo=None).isoformat(sep=" ") + marker + db_session.connection().exec_driver_sql( + "UPDATE audit_events SET occurred_at = ? WHERE id = ?", + (malformed_timestamp, event_id), + ) + db_session.commit() + caplog.set_level("ERROR") + + prometheus = client.get("/api/v1/metrics/prometheus") + history = client.get("/api/v1/tasks/history?window_hours=1") + + assert prometheus.status_code == 500 + assert prometheus.json() == { + "detail": { + "code": "audit_event_integrity_error", + "message": "A retained audit event failed integrity validation", + } + } + assert history.status_code == 500 + assert history.json() == { + "detail": { + "code": "audit_event_integrity_error", + "message": "A retained audit event failed integrity validation", + } + } + assert marker not in prometheus.text + assert marker not in history.text + assert marker not in caplog.text + + +def test_prometheus_database_and_renderer_failures_are_fixed_and_fail_closed( + client, + monkeypatch, + caplog, +) -> None: + database_secret = "postgresql://user:password@database/private" + original_collector = observability_api._collect_database_snapshot + + def fail_database(_settings): + raise SQLAlchemyError(database_secret) + + monkeypatch.setattr(observability_api, "_collect_database_snapshot", fail_database) + caplog.set_level("ERROR", logger="app.api.v1.observability") + database_response = client.get("/api/v1/metrics/prometheus") + + assert database_response.status_code == 503 + assert database_response.json()["detail"]["code"] == "metrics_database_unavailable" + assert database_secret not in database_response.text + assert database_secret not in caplog.text + + monkeypatch.setattr( + observability_api, + "_collect_database_snapshot", + original_collector, + ) + + def fail_renderer(*_args, **_kwargs): + raise PrometheusRenderingError("sk-renderer-secret") + + monkeypatch.setattr(observability_api, "render_prometheus", fail_renderer) + renderer_response = client.get("/api/v1/metrics/prometheus") + + assert renderer_response.status_code == 500 + assert renderer_response.json()["detail"]["code"] == "metrics_rendering_error" + assert "sk-renderer-secret" not in renderer_response.text + assert "sk-renderer-secret" not in caplog.text + + +def test_prometheus_collection_is_single_flight_per_api_process(client, monkeypatch) -> None: + entered = threading.Event() + release = threading.Event() + original = observability_api._collect_database_snapshot + + def blocked(settings): + entered.set() + assert release.wait(timeout=2) + return original(settings) + + monkeypatch.setattr(observability_api, "_collect_database_snapshot", blocked) + with ThreadPoolExecutor(max_workers=1) as executor: + first_future = executor.submit(client.get, "/api/v1/metrics/prometheus") + assert entered.wait(timeout=1) + second = client.get("/api/v1/metrics/prometheus") + release.set() + first = first_future.result(timeout=3) + + assert second.status_code == 429 + assert second.json()["detail"]["code"] == "metrics_scrape_in_progress" + assert first.status_code == 200 + + +@pytest.mark.asyncio +async def test_prometheus_collection_gate_survives_repeated_request_cancellation( + client, + monkeypatch, +) -> None: + entered = threading.Event() + release = threading.Event() + original = observability_api._collect_database_snapshot + + def blocked(settings): + entered.set() + assert release.wait(timeout=2) + return original(settings) + + monkeypatch.setattr(observability_api, "_collect_database_snapshot", blocked) + request = SimpleNamespace( + url=SimpleNamespace(query=""), + app=client.app, + ) + first = asyncio.create_task(observability_api.prometheus_metrics(request, get_settings())) + + try: + assert await asyncio.to_thread(entered.wait, 1) + first.cancel() + await asyncio.sleep(0) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + + assert observability_api._COLLECTION_LOCK.locked() + with pytest.raises(HTTPException) as overlapping: + await observability_api.prometheus_metrics(request, get_settings()) + assert overlapping.value.status_code == 429 + assert overlapping.value.detail["code"] == "metrics_scrape_in_progress" + finally: + release.set() + for _ in range(100): + if not observability_api._COLLECTION_LOCK.locked(): + break + await asyncio.sleep(0.01) + + assert not observability_api._COLLECTION_LOCK.locked() + + +def test_prometheus_audit_cap_uses_limit_plus_one_and_never_truncates_counts( + client, + db_session, +) -> None: + now = _database_now() + for index in range(2): + append_audit_event( + db_session, + event_key=f"metrics:cap:{index}", + event_type="run_claimed", + occurred_at=now - timedelta(seconds=index + 1), + payload={"dispatch_count": index}, + run_id=f"metrics-cap-run-{index}", + worker_id="metrics-cap-worker", + attempt=1, + lease_token=1, + ) + db_session.commit() + + with SessionLocal() as session, session.begin(): + configure_read_snapshot(session) + window_end = database_clock(session) + with pytest.raises(MetricsObservationLimitExceeded): + collect_task_history( + session, + window_start=window_end - timedelta(minutes=15), + window_end=window_end, + audit_event_limit=1, + ) + assert METRICS_AUDIT_EVENT_LIMIT == 50_000 + + +def test_prometheus_renderer_rejects_negative_or_nonfinite_snapshot_values(client) -> None: + snapshot = observability_api._collect_database_snapshot(get_settings()) + + with pytest.raises(PrometheusRenderingError): + render_prometheus( + replace(snapshot, current=replace(snapshot.current, pending=-1)), + queue_configured=False, + queue_available=False, + recovery_alert_seconds=60, + ) + with pytest.raises(PrometheusRenderingError): + render_prometheus( + snapshot, + queue_configured=False, + queue_available=False, + recovery_alert_seconds=float("inf"), + ) diff --git a/backend/tests/test_sqlite_postgres_import.py b/backend/tests/test_sqlite_postgres_import.py index ada9008..9719e50 100644 --- a/backend/tests/test_sqlite_postgres_import.py +++ b/backend/tests/test_sqlite_postgres_import.py @@ -4,6 +4,7 @@ import hashlib import io +import json import os import re import sqlite3 @@ -43,6 +44,7 @@ ) from app.db.session import create_database_engine from app.db.types import UTCDateTime +from app.governance.repository import _policy_hash from app.models import CredentialSource, ProviderType, QuestionType, RunStatus from app.security import CredentialKeyring, EncryptedCredential @@ -98,6 +100,57 @@ def _insert_complete_mock_evidence( question_execution_id = f"question-execution-import{suffix}" reservation_id = f"reservation-import{suffix}" window_start = created_at.replace(second=0, microsecond=0) + policy_values = { + "global_concurrency_limit": 8, + "provider_concurrency_limit": 4, + "model_concurrency_limit": 2, + "run_concurrency_limit": 1, + "global_requests_per_minute": 120, + "provider_requests_per_minute": 60, + "model_requests_per_minute": 30, + "run_requests_per_minute": 10, + "global_tokens_per_minute": 100_000, + "provider_tokens_per_minute": 50_000, + "model_tokens_per_minute": 25_000, + "run_tokens_per_minute": 5_000, + "global_lifetime_request_budget": 10_000, + "global_lifetime_token_budget": 1_000_000, + "global_lifetime_cost_budget_usd": Decimal("100.00000000"), + "run_lifetime_request_budget": 10, + "run_lifetime_token_budget": 10_000, + "run_lifetime_cost_budget_usd": Decimal("1.00000000"), + "backlog_limit": 32, + "question_quantum": 2, + } + policy_hash = _policy_hash(policy_values) + governance_snapshot = { + "policy_id": policy_id, + "policy_version": 1, + "policy_hash": policy_hash, + **{ + name: format(value, "f") if isinstance(value, Decimal) else value + for name, value in policy_values.items() + }, + "provider_scope_key": hashlib.sha256(b"mock\0offline").hexdigest(), + "local_admission_only": True, + "run_overrides": { + "input_token_reservation": 64, + "lifetime_request_budget": 10, + "lifetime_token_budget": 10_000, + "lifetime_cost_budget_usd": "1.00000000", + }, + } + audit_payload = { + "disposition": "settled_actual", + "outcome": "succeeded", + "input_tokens": 8, + "output_tokens": 3, + "cost_usd": "0.00000050", + "reconciled": False, + } + audit_payload_hash = hashlib.sha256( + json.dumps(audit_payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() engine = create_database_engine(_sqlite_url(path)) tables = Base.metadata.tables encrypted = CredentialKeyring( @@ -165,32 +218,40 @@ def _insert_complete_mock_evidence( { "id": policy_id, "version": 1, - "policy_hash": "c" * 64, + "policy_hash": policy_hash, "is_active": True, - "global_concurrency_limit": 8, - "provider_concurrency_limit": 4, - "model_concurrency_limit": 2, - "run_concurrency_limit": 1, - "global_requests_per_minute": 120, - "provider_requests_per_minute": 60, - "model_requests_per_minute": 30, - "run_requests_per_minute": 10, - "global_tokens_per_minute": 100_000, - "provider_tokens_per_minute": 50_000, - "model_tokens_per_minute": 25_000, - "run_tokens_per_minute": 5_000, - "global_lifetime_request_budget": 10_000, - "global_lifetime_token_budget": 1_000_000, - "global_lifetime_cost_budget_usd": Decimal("100.00000000"), - "run_lifetime_request_budget": 10, - "run_lifetime_token_budget": 10_000, - "run_lifetime_cost_budget_usd": Decimal("1.00000000"), - "backlog_limit": 32, - "question_quantum": 2, + **policy_values, "activated_at": created_at, "created_at": created_at, }, ) + connection.execute( + tables["worker_processes"].insert(), + [ + { + "generation_id": str(uuid4()), + "worker_id": f"worker-import-stale{suffix}", + "started_at": created_at, + "last_seen_at": created_at, + "last_scan_at": created_at, + "last_claim_at": None, + "last_progress_at": None, + "last_lease_heartbeat_at": None, + "stopped_at": None, + }, + { + "generation_id": str(uuid4()), + "worker_id": f"worker-import-stopped{suffix}", + "started_at": created_at, + "last_seen_at": finished_at, + "last_scan_at": finished_at, + "last_claim_at": finished_at, + "last_progress_at": finished_at, + "last_lease_heartbeat_at": finished_at, + "stopped_at": finished_at, + }, + ], + ) connection.execute( tables["benchmarks"].insert(), { @@ -283,6 +344,7 @@ def _insert_complete_mock_evidence( "execution": { "restart_recovery": "database_lease_resume_missing_responses" }, + "governance": governance_snapshot, }, "benchmark_hash_snapshot": "a" * 64, "prompt_template_snapshot": {"system": "Answer exactly", "suffix": ""}, @@ -456,8 +518,8 @@ def _insert_complete_mock_evidence( "id": f"audit-import{suffix}", "event_key": f"reservation:{reservation_id}:settled_actual", "event_type": "provider_attempt_settled", - "payload_hash": "d" * 64, - "payload": {"disposition": "settled_actual", "requests": 1}, + "payload_hash": audit_payload_hash, + "payload": audit_payload, "retention_class": "operational", "occurred_at": created_at + timedelta(seconds=1), "expires_at": created_at + timedelta(days=90, seconds=1), @@ -675,6 +737,7 @@ def test_read_only_source_preflight_preserves_file_and_reconciles_rows(tmp_path: assert snapshot.summaries["question_executions"].row_count == 1 assert snapshot.summaries["provider_call_reservations"].row_count == 1 assert snapshot.summaries["audit_events"].row_count == 1 + assert snapshot.summaries["worker_processes"].row_count == 2 assert snapshot.summaries["questions"].row_count == 2 assert snapshot.summaries["evaluation_responses"].row_count == 2 assert hashlib.sha256(source.read_bytes()).hexdigest() == before @@ -799,6 +862,105 @@ def test_source_preflight_rejects_active_provider_reservations( engine.dispose() +def test_source_preflight_rejects_live_worker_but_accepts_stale_and_stopped( + tmp_path: Path, +) -> None: + source = tmp_path / "worker-process-state.db" + _run_alembic(source) + _insert_complete_mock_evidence(source) + + snapshot = _read_only_snapshot(source) + assert snapshot.summaries["worker_processes"].row_count == 2 + + with sqlite3.connect(source) as connection: + connection.execute( + "UPDATE worker_processes SET last_seen_at = CURRENT_TIMESTAMP, " + "last_scan_at = CURRENT_TIMESTAMP WHERE stopped_at IS NULL" + ) + + engine = _read_only_sqlite_engine(source) + try: + with engine.connect() as connection: + connection.exec_driver_sql("BEGIN") + with pytest.raises(SQLiteImportError, match="live Worker generation"): + preflight_sqlite_source(connection, worker_stale_seconds=60) + finally: + engine.dispose() + + +@pytest.mark.parametrize( + ("damage", "error_pattern"), + [ + ("policy_hash", "governance policy or managed Run snapshot"), + ("policy_column", "governance policy or managed Run snapshot"), + ("policy_snapshot", "governance policy or managed Run snapshot"), + ("run_override", "governance policy or managed Run snapshot"), + ("audit_payload_hash", "retained audit event failed integrity validation"), + ], +) +def test_import_rejects_governance_and_audit_integrity_drift_before_target_access( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + damage: str, + error_pattern: str, +) -> None: + source = tmp_path / f"source-integrity-{damage}.db" + _run_alembic(source) + _insert_complete_mock_evidence(source) + with sqlite3.connect(source) as connection: + if damage == "policy_hash": + connection.execute( + "UPDATE governance_policies SET policy_hash = ? WHERE id = 'policy-import'", + ("e" * 64,), + ) + elif damage == "policy_column": + connection.execute( + "UPDATE governance_policies SET question_quantum = 3 WHERE id = 'policy-import'" + ) + elif damage in {"policy_snapshot", "run_override"}: + encoded = connection.execute( + "SELECT model_parameters_snapshot FROM evaluation_runs WHERE id = 'run-import'" + ).fetchone()[0] + snapshot = json.loads(encoded) + if damage == "policy_snapshot": + snapshot["governance"]["policy_hash"] = "e" * 64 + else: + snapshot["governance"]["run_overrides"]["input_token_reservation"] = 65 + connection.execute( + "UPDATE evaluation_runs SET model_parameters_snapshot = ? WHERE id = 'run-import'", + (json.dumps(snapshot, sort_keys=True, separators=(",", ":")),), + ) + else: + connection.execute( + "UPDATE audit_events SET payload_hash = ? WHERE id = 'audit-import'", + ("e" * 64,), + ) + + class TargetProbe: + connect_calls = 0 + + def connect(self): + self.connect_calls += 1 + raise AssertionError("damaged source must not open the target database") + + def dispose(self) -> None: + return None + + target = TargetProbe() + monkeypatch.setattr(import_sqlite_module, "create_database_engine", lambda _url: target) + output = io.StringIO() + + with pytest.raises(SQLiteImportError, match=error_pattern): + import_sqlite_to_postgres( + str(source), + "postgresql+psycopg://llmbenchlab@localhost/empty_target", + output=output, + ) + + assert target.connect_calls == 0 + assert output.getvalue() == "" + + @pytest.mark.parametrize( "statement", [ @@ -1039,8 +1201,12 @@ def test_copy_snapshot_preserves_all_governance_tables_in_sqlite(tmp_path: Path) "0.00000050" ) assert postcommit.rows["audit_events"][0]["payload"] == { + "cost_usd": "0.00000050", "disposition": "settled_actual", - "requests": 1, + "input_tokens": 8, + "outcome": "succeeded", + "output_tokens": 3, + "reconciled": False, } assert len(postcommit.rows["model_credentials"]) == 1 assert postcommit.rows["model_credentials"][0] == source_snapshot.rows["model_credentials"][0] diff --git a/backend/tests/test_task_queue.py b/backend/tests/test_task_queue.py index 68135ee..5235eac 100644 --- a/backend/tests/test_task_queue.py +++ b/backend/tests/test_task_queue.py @@ -9,6 +9,11 @@ from app.task_queue import TASK_MESSAGE_VERSION, QueueUnavailable, RedisRunQueue +RUN_ID = "00000000-0000-4000-8000-000000000001" +CORRELATION_ID = "00000000-0000-4000-8000-000000000002" +OTHER_RUN_ID = "00000000-0000-4000-8000-000000000003" +OTHER_CORRELATION_ID = "00000000-0000-4000-8000-000000000004" + class _FakeRedis: def __init__(self) -> None: @@ -72,7 +77,7 @@ async def test_publish_uses_versioned_bounded_stream_message() -> None: client = _FakeRedis() queue = _queue(client) - message_id = await queue.publish("run-1", correlation_id="correlation-1") + message_id = await queue.publish(RUN_ID, correlation_id=CORRELATION_ID) assert message_id == "1-0" assert client.added == [ @@ -81,8 +86,8 @@ async def test_publish_uses_versioned_bounded_stream_message() -> None: "runs", { "version": TASK_MESSAGE_VERSION, - "run_id": "run-1", - "correlation_id": "correlation-1", + "run_id": RUN_ID, + "correlation_id": CORRELATION_ID, }, ), {"maxlen": 100, "approximate": True}, @@ -97,7 +102,42 @@ async def test_publish_timeout_is_sanitized_and_bounded() -> None: queue = _queue(client, publish_timeout=0.01) with pytest.raises(QueueUnavailable, match="queue_publish_unavailable"): - await queue.publish("run-1", correlation_id="correlation-1") + await queue.publish(RUN_ID, correlation_id=CORRELATION_ID) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["run_id", "correlation_id"]) +async def test_redis_delivery_rejects_secret_canary_identifiers(field: str) -> None: + client = _FakeRedis() + queue = _queue(client) + secret = "sk-redis-queue-canary-must-not-reach-worker-logs" + fields = { + "version": TASK_MESSAGE_VERSION, + "run_id": RUN_ID, + "correlation_id": CORRELATION_ID, + } + fields[field] = secret + client.read_response = [("runs", [("2-0", fields)])] + + delivery = await queue.read_new(consumer="worker-1", block_milliseconds=0) + + assert delivery is not None and not delivery.is_valid + assert delivery.run_id is None + assert delivery.correlation_id is None + assert secret not in repr(delivery) + + +@pytest.mark.asyncio +async def test_publish_rejects_non_uuid_identifiers_before_redis_call() -> None: + client = _FakeRedis() + queue = _queue(client) + + with pytest.raises(ValueError, match="canonical UUIDs"): + await queue.publish("sk-invalid-run", correlation_id=CORRELATION_ID) + with pytest.raises(ValueError, match="canonical UUIDs"): + await queue.publish(RUN_ID, correlation_id="sk-invalid-correlation") + + assert client.added == [] @pytest.mark.asyncio @@ -133,8 +173,8 @@ async def test_read_claim_cursor_validation_ack_and_close() -> None: "2-0", { "version": TASK_MESSAGE_VERSION, - "run_id": "run-2", - "correlation_id": "correlation-2", + "run_id": RUN_ID, + "correlation_id": CORRELATION_ID, }, ) ], @@ -143,7 +183,7 @@ async def test_read_claim_cursor_validation_ack_and_close() -> None: delivery = await queue.read_new(consumer="worker-1", block_milliseconds=0) assert delivery is not None and delivery.is_valid assert delivery.message_id == "2-0" - assert delivery.run_id == "run-2" + assert delivery.run_id == RUN_ID client.claim_response = [ "7-0", @@ -152,8 +192,8 @@ async def test_read_claim_cursor_validation_ack_and_close() -> None: "3-0", { "version": "unknown-version", - "run_id": "run-3", - "correlation_id": "correlation-3", + "run_id": OTHER_RUN_ID, + "correlation_id": OTHER_CORRELATION_ID, }, ) ], diff --git a/backend/tests/test_worker.py b/backend/tests/test_worker.py index ab70377..4574834 100644 --- a/backend/tests/test_worker.py +++ b/backend/tests/test_worker.py @@ -12,6 +12,7 @@ from app.db.session import SessionLocal from app.runners.run_leases import ReapReport from app.task_queue import QueueUnavailable, RunTaskDelivery +from app.worker_progress import WorkerProgressEvent from app.workers import WorkerService @@ -126,6 +127,23 @@ async def close(self) -> None: self.closed = True +class _FakeProgressRecorder: + def __init__(self) -> None: + self.events: list[WorkerProgressEvent] = [] + self.started = False + self.stopped = False + + def note(self, event: WorkerProgressEvent) -> None: + self.events.append(event) + + async def start(self) -> None: + self.started = True + + async def stop(self) -> bool: + self.stopped = True + return True + + def _settings(**updates): return get_settings().model_copy( update={ @@ -136,7 +154,7 @@ def _settings(**updates): ) -def _service(repository, runner, queue) -> WorkerService: +def _service(repository, runner, queue, *, progress=None) -> WorkerService: return WorkerService( SessionLocal, _settings(), @@ -144,6 +162,7 @@ def _service(repository, runner, queue) -> WorkerService: worker_id="worker-test", lease_repository=repository, runner=runner, + progress_recorder=progress or _FakeProgressRecorder(), ) @@ -167,7 +186,9 @@ def assert_not_acked() -> None: @pytest.mark.asyncio async def test_worker_does_not_ack_unhandled_runner_failure(caplog) -> None: - queue = _FakeQueue([RunTaskDelivery("2-0", "run-2", "correlation-2")]) + run_id = "00000000-0000-4000-8000-000000000001" + correlation_id = "00000000-0000-4000-8000-000000000002" + queue = _FakeQueue([RunTaskDelivery("2-0", run_id, correlation_id)]) runner = _FakeRunner() runner.raise_error = True service = _service(_FakeRepository(), runner, queue) @@ -175,13 +196,13 @@ async def test_worker_does_not_ack_unhandled_runner_failure(caplog) -> None: assert await service.run_once() is True - assert runner.calls == ["run-2"] + assert runner.calls == [run_id] assert queue.acked == [] failure = next( record for record in caplog.records if record.event == "worker_run_unhandled_error" ) - assert failure.run_id == "run-2" - assert failure.correlation_id == "correlation-2" + assert failure.run_id == run_id + assert failure.correlation_id == correlation_id assert failure.result == "not_acknowledged" @@ -224,6 +245,31 @@ async def test_temporary_database_failure_does_not_destroy_worker_service() -> N assert runner.calls == ["run-after-recovery"] +@pytest.mark.asyncio +async def test_scan_progress_requires_both_reaper_and_due_scan_success() -> None: + progress = _FakeProgressRecorder() + service = _service(_FakeRepository(), _FakeRunner(), None, progress=progress) + + assert await service.run_once() is False + assert progress.events == [WorkerProgressEvent.SCAN] + + progress.events.clear() + repository = _FakeRepository() + repository.fail_reap_once = True + service = _service(repository, _FakeRunner(), None, progress=progress) + assert await service.run_once() is False + assert progress.events == [] + + class _FailDueRepository(_FakeRepository): + def due_run_ids(self, *, limit: int) -> tuple[str, ...]: + del limit + raise SQLAlchemyError("controlled") + + service = _service(_FailDueRepository(), _FakeRunner(), None, progress=progress) + assert await service.run_once() is False + assert progress.events == [] + + @pytest.mark.asyncio async def test_stop_during_blocking_queue_read_never_starts_new_run() -> None: queue = _FakeQueue([RunTaskDelivery("4-0", "run-late", "correlation-4")]) @@ -252,11 +298,15 @@ def due_run_ids(self, *, limit: int) -> tuple[str, ...]: return ("run-after-stop",) runner = _FakeRunner() - service = _service(_StoppingRepository(), runner, None) + progress = _FakeProgressRecorder() + service = _service(_StoppingRepository(), runner, None, progress=progress) await asyncio.wait_for(service.run(stop), timeout=1) assert runner.calls == [] + assert progress.events == [WorkerProgressEvent.SCAN] + assert progress.started is True + assert progress.stopped is True @pytest.mark.asyncio diff --git a/backend/tests/test_worker_probe.py b/backend/tests/test_worker_probe.py index 2141bce..e04c001 100644 --- a/backend/tests/test_worker_probe.py +++ b/backend/tests/test_worker_probe.py @@ -46,7 +46,12 @@ def fail_database(): assert await probe_module._probe() == 1 output = capsys.readouterr().out - assert json.loads(output) == {"status": "not_ready", "database": "unavailable"} + assert json.loads(output) == { + "status": "not_ready", + "database": "unavailable", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", + } assert secret not in output assert engine.disposed is True @@ -71,6 +76,8 @@ async def test_worker_probe_reports_queue_degradation_but_keeps_reconciliation_r "schema": "ok", "queue": "unavailable", "database_reconciliation": "available", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", } assert queue.closed is True assert engine.disposed is True @@ -99,6 +106,8 @@ def fail_queue(_settings): "database": "ok", "schema": "ok", "queue": "configuration_error", + "probe_scope": "dependencies_only", + "main_loop_progress": "not_checked", } assert secret not in output assert engine.disposed is True diff --git a/backend/tests/test_worker_progress.py b/backend/tests/test_worker_progress.py new file mode 100644 index 0000000..63027ba --- /dev/null +++ b/backend/tests/test_worker_progress.py @@ -0,0 +1,240 @@ +"""Worker generation registration, coalescing, and stale read-model tests.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime, timedelta +from threading import Event + +import pytest +from pydantic import ValidationError +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import sessionmaker + +from app.core.config import Settings +from app.db.base import Base +from app.db.session import create_database_engine +from app.models import WorkerProcess +from app.worker_progress import ( + WorkerProgressEvent, + WorkerProgressRecorder, + WorkerProgressStateError, + _PendingEvent, + collect_worker_progress, +) + + +@pytest.fixture +def worker_store(tmp_path): + engine = create_database_engine(f"sqlite:///{tmp_path / 'worker-progress.db'}") + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield factory + finally: + engine.dispose() + + +@pytest.mark.asyncio +async def test_recorder_uses_one_db_timestamp_for_coalesced_events_and_zero_idle_writes( + worker_store, +) -> None: + recorder = WorkerProgressRecorder( + worker_store, + worker_id="worker-progress-test", + generation_id="00000000-0000-0000-0000-000000000001", + flush_seconds=60, + ) + + await recorder.start() + with worker_store() as session: + registered = session.get(WorkerProcess, recorder.generation_id) + assert registered is not None + registered_seen_at = registered.last_seen_at + assert registered.started_at == registered_seen_at + assert registered.stopped_at is None + + assert await recorder.flush_now() is False + with worker_store() as session: + row = session.get(WorkerProcess, recorder.generation_id) + assert row is not None and row.last_seen_at == registered_seen_at + + for event in WorkerProgressEvent: + recorder.note(event) + recorder.note(event) + assert sorted(await asyncio.gather(recorder.flush_now(), recorder.flush_now())) == [False, True] + + with worker_store() as session: + row = session.get(WorkerProcess, recorder.generation_id) + assert row is not None + assert row.last_seen_at >= registered_seen_at + assert { + row.last_scan_at, + row.last_claim_at, + row.last_progress_at, + row.last_lease_heartbeat_at, + } == {row.last_seen_at} + + assert await recorder.stop() is True + with worker_store() as session: + row = session.get(WorkerProcess, recorder.generation_id) + assert row is not None + assert row.stopped_at is not None + assert row.stopped_at >= row.last_seen_at + + recorder.note(WorkerProgressEvent.SCAN) + assert await recorder.flush_now() is False + with pytest.raises(WorkerProgressStateError): + recorder._flush(_PendingEvent.SCAN) + + +@pytest.mark.asyncio +async def test_failed_flush_retains_pending_event_for_retry(worker_store, monkeypatch) -> None: + recorder = WorkerProgressRecorder( + worker_store, + worker_id="worker-progress-retry", + generation_id="00000000-0000-0000-0000-000000000002", + flush_seconds=60, + ) + await recorder.start() + recorder.note(WorkerProgressEvent.CLAIM) + original_flush = recorder._flush + + def fail_once(_pending): + raise RuntimeError("controlled") + + monkeypatch.setattr(recorder, "_flush", fail_once) + assert await recorder.flush_now() is False + monkeypatch.setattr(recorder, "_flush", original_flush) + assert await recorder.flush_now() is True + await recorder.stop() + + with worker_store() as session: + row = session.get(WorkerProcess, recorder.generation_id) + assert row is not None and row.last_claim_at is not None + + +@pytest.mark.asyncio +async def test_stop_persists_preexisting_bits_and_drops_late_notes( + worker_store, + monkeypatch, +) -> None: + recorder = WorkerProgressRecorder( + worker_store, + worker_id="worker-progress-stop-race", + generation_id="00000000-0000-0000-0000-000000000003", + flush_seconds=60, + ) + await recorder.start() + recorder.note(WorkerProgressEvent.SCAN) + stop_entered = Event() + release_stop = Event() + original_stop = recorder._stop + + def controlled_stop(pending): + stop_entered.set() + assert release_stop.wait(timeout=2) + original_stop(pending) + + monkeypatch.setattr(recorder, "_stop", controlled_stop) + stopping = asyncio.create_task(recorder.stop()) + assert await asyncio.to_thread(stop_entered.wait, 2) + recorder.note(WorkerProgressEvent.CLAIM) + release_stop.set() + assert await stopping is True + + with worker_store() as session: + row = session.get(WorkerProcess, recorder.generation_id) + assert row is not None + assert row.last_scan_at == row.last_seen_at + assert row.last_claim_at is None + assert row.stopped_at == row.last_seen_at + + +def test_worker_progress_snapshot_uses_inclusive_live_boundary_and_ignores_stopped( + worker_store, +) -> None: + now = datetime(2026, 8, 28, 8, 0, tzinfo=UTC) + cutoff = now - timedelta(seconds=60) + with worker_store() as session, session.begin(): + session.add_all( + [ + WorkerProcess( + generation_id="00000000-0000-0000-0000-000000000011", + worker_id="worker-live-boundary", + started_at=cutoff - timedelta(seconds=1), + last_seen_at=cutoff, + last_scan_at=cutoff, + ), + WorkerProcess( + generation_id="00000000-0000-0000-0000-000000000012", + worker_id="worker-stalled", + started_at=cutoff - timedelta(seconds=10), + last_seen_at=cutoff - timedelta(microseconds=1), + last_progress_at=cutoff - timedelta(microseconds=1), + ), + WorkerProcess( + generation_id="00000000-0000-0000-0000-000000000013", + worker_id="worker-stopped", + started_at=cutoff, + last_seen_at=now, + last_claim_at=now, + stopped_at=now, + ), + ] + ) + + with worker_store() as session: + snapshot = collect_worker_progress( + session, + now=now, + stale_seconds=60, + expected=3, + ) + + assert snapshot.registered == 2 + assert snapshot.live == 1 + assert snapshot.stalled == 1 + assert snapshot.shortfall == 2 + assert snapshot.last_seen_at == cutoff + assert snapshot.last_scan_at == cutoff + assert snapshot.last_progress_at == cutoff - timedelta(microseconds=1) + assert snapshot.last_claim_at is None + + +def test_worker_timing_requires_stale_threshold_to_cover_every_cadence() -> None: + with pytest.raises(ValidationError, match="worker_progress_stale_seconds"): + Settings( + _env_file=None, + worker_heartbeat_seconds=10, + worker_lease_seconds=30, + worker_progress_flush_seconds=5, + worker_progress_stale_seconds=29, + ) + + settings = Settings( + _env_file=None, + worker_heartbeat_seconds=10, + worker_lease_seconds=30, + worker_progress_stale_seconds=30, + worker_expected_processes=0, + ) + assert settings.worker_expected_processes == 0 + + +def test_worker_process_constraints_reject_event_after_seen(worker_store) -> None: + now = datetime(2026, 8, 28, 8, 0, tzinfo=UTC) + with pytest.raises(IntegrityError), worker_store() as session, session.begin(): + session.add( + WorkerProcess( + generation_id="00000000-0000-0000-0000-000000000021", + worker_id="worker-invalid", + started_at=now, + last_seen_at=now, + last_scan_at=now + timedelta(seconds=1), + ) + ) + + with worker_store() as session: + assert session.scalar(select(WorkerProcess.generation_id)) is None diff --git a/compose.yaml b/compose.yaml index a2bc94b..8c3db5e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,10 @@ x-app-environment: &app-environment LLMBENCHLAB_WORKER_RETRY_BACKOFF_BASE_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_RETRY_BACKOFF_BASE_SECONDS:-1} LLMBENCHLAB_WORKER_RETRY_BACKOFF_CAP_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_RETRY_BACKOFF_CAP_SECONDS:-30} LLMBENCHLAB_WORKER_SHUTDOWN_GRACE_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_SHUTDOWN_GRACE_SECONDS:-30} + LLMBENCHLAB_WORKER_PROGRESS_FLUSH_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_PROGRESS_FLUSH_SECONDS:-5} + LLMBENCHLAB_WORKER_PROGRESS_STALE_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_PROGRESS_STALE_SECONDS:-60} + LLMBENCHLAB_WORKER_EXPECTED_PROCESSES: ${LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES:-1} + LLMBENCHLAB_WORKER_RECOVERY_ALERT_SECONDS: ${LLMBENCHLAB_COMPOSE_WORKER_RECOVERY_ALERT_SECONDS:-60} LLMBENCHLAB_REDIS_BLOCK_MILLISECONDS: ${LLMBENCHLAB_COMPOSE_REDIS_BLOCK_MILLISECONDS:-1000} LLMBENCHLAB_REDIS_OPERATION_TIMEOUT_SECONDS: ${LLMBENCHLAB_COMPOSE_REDIS_OPERATION_TIMEOUT_SECONDS:-1} LLMBENCHLAB_DATABASE_POOL_SIZE: ${LLMBENCHLAB_COMPOSE_DATABASE_POOL_SIZE:-5} diff --git a/deploy/observability/prometheus-alerts.json b/deploy/observability/prometheus-alerts.json new file mode 100644 index 0000000..32abd3f --- /dev/null +++ b/deploy/observability/prometheus-alerts.json @@ -0,0 +1,130 @@ +{ + "groups": [ + { + "name": "llmbenchlab.phase2", + "interval": "30s", + "rules": [ + { + "alert": "LLMBenchLabMetricsUnavailable", + "expr": "up{job=\"llmbenchlab\"} == 0", + "for": "2m", + "labels": { + "severity": "critical", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab metrics exporter is unavailable", + "description": "Prometheus has been unable to scrape the controlled metrics endpoint for two minutes.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-exporter-unavailable", + "silence_policy": "Narrow job or instance matcher only; owner, ticket, and expiry required; maximum 4h; never global." + } + }, + { + "alert": "LLMBenchLabBacklogPersistent", + "expr": "llmbenchlab_runs_due_pending > 0", + "for": "15m", + "labels": { + "severity": "warning", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab dispatchable backlog is persistent", + "description": "At least one due pending Run has remained dispatchable for fifteen minutes.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-backlog-persistent", + "silence_policy": "Narrow job matcher only; owner, ticket, and expiry required; maximum 4h; never global." + } + }, + { + "alert": "LLMBenchLabDeadLettered", + "expr": "llmbenchlab_audit_events_window{event_type=\"run_dead_lettered\"} > 0", + "for": "0s", + "labels": { + "severity": "critical", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab recorded a dead-letter event", + "description": "A validated run_dead_lettered event is present in the fixed fifteen-minute audit window.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-dead-letter", + "silence_policy": "Narrow job matcher only after evidence capture; owner, ticket, and expiry required; maximum 1h." + } + }, + { + "alert": "LLMBenchLabGovernanceIntegrityError", + "expr": "llmbenchlab_audit_events_window{event_type=\"governance_integrity_error\"} > 0", + "for": "0s", + "labels": { + "severity": "critical", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab recorded a governance integrity error", + "description": "A validated governance_integrity_error event is present in the fixed fifteen-minute audit window.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-governance-integrity", + "silence_policy": "Do not routinely silence; capture evidence and remediate the durable governance facts." + } + }, + { + "alert": "LLMBenchLabGovernanceOverdrawn", + "expr": "llmbenchlab_governance_scopes_overdrawn > 0", + "for": "0s", + "labels": { + "severity": "critical", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab governance scope is overdrawn", + "description": "At least one database governance scope is marked overdrawn.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-governance-overdraw", + "silence_policy": "Do not routinely silence; capture evidence and reconcile the durable governance ledger." + } + }, + { + "alert": "LLMBenchLabQueueDegraded", + "expr": "llmbenchlab_queue_configured == 1 and llmbenchlab_queue_available == 0", + "for": "2m", + "labels": { + "severity": "warning", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab Redis notification queue is degraded", + "description": "The configured non-authoritative Redis notification queue has been unavailable for two minutes.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-queue-degraded", + "silence_policy": "Narrow job matcher only; owner, ticket, and expiry required; maximum 4h; never global." + } + }, + { + "alert": "LLMBenchLabWorkerStalled", + "expr": "llmbenchlab_worker_shortfall > 0", + "for": "2m", + "labels": { + "severity": "critical", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab has fewer live Workers than configured", + "description": "Database-time Worker progress shows a live process shortfall for two minutes.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-worker-stalled", + "silence_policy": "Narrow job matcher only during controlled maintenance; owner, ticket, and expiry required; maximum 1h." + } + }, + { + "alert": "LLMBenchLabLeaseRecoverySlow", + "expr": "llmbenchlab_run_expired_lease_oldest_age_seconds > llmbenchlab_run_recovery_alert_threshold_seconds", + "for": "1m", + "labels": { + "severity": "warning", + "component": "llmbenchlab-control-plane" + }, + "annotations": { + "summary": "LLMBenchLab expired lease recovery is slow", + "description": "The oldest currently expired running lease exceeds the configured recovery alert threshold.", + "runbook_url": "https://github.com/CWNU-Open-Source-Community/LLMBenchLab/blob/main/docs/OPERATIONS.md#alert-lease-recovery-slow", + "silence_policy": "Narrow job matcher only after recovery ownership is established; owner, ticket, and expiry required; maximum 15m." + } + } + ] + } + ] +} diff --git a/deploy/observability/prometheus-scrape.example.yml b/deploy/observability/prometheus-scrape.example.yml new file mode 100644 index 0000000..b8a6057 --- /dev/null +++ b/deploy/observability/prometheus-scrape.example.yml @@ -0,0 +1,7 @@ +scrape_configs: + - job_name: llmbenchlab + scrape_interval: 60s + metrics_path: /api/v1/metrics/prometheus + static_configs: + - targets: + - api:8000 diff --git a/docs/API.md b/docs/API.md index 23c5330..a4423b6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -120,6 +120,7 @@ Benchmark 校验错误还会提供文件、行、列或 JSON Pointer： | GET | `/ready` | 200/503 | 数据库、Alembic head 和 Redis 组件就绪状态 | | GET | `/tasks/metrics` | 200 | 数据库当前任务 gauges | | GET | `/tasks/history` | 200 | 有界 UTC 窗口内的类型化事件 counters 与 Run 延迟分位数 | +| GET | `/metrics/prometheus` | 200 | 固定低基数 Prometheus text exposition；不接受查询参数 | | GET | `/info` | 200 | 服务、协议和能力信息 | | GET | `/governance/policy` | 200/404 | 只读查看当前治理策略；未初始化时不产生写入 | | PUT | `/governance/policy` | 200 | 原子应用一份完整、版本化的治理策略 | @@ -264,6 +265,17 @@ curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics "total_attempts": 3, "total_failed_attempts": 1, "total_dispatches": 4, + "worker_expected_processes": 2, + "worker_registered_processes": 2, + "worker_live_processes": 2, + "worker_stalled_processes": 0, + "worker_shortfall_processes": 0, + "worker_stale_after_seconds": 60.0, + "worker_last_seen_at": "2026-08-25T05:59:59Z", + "worker_last_scan_at": "2026-08-25T05:59:58Z", + "worker_last_claim_at": "2026-08-25T05:59:50Z", + "worker_last_progress_at": "2026-08-25T05:59:55Z", + "worker_last_lease_heartbeat_at": "2026-08-25T05:59:57Z", "timestamp": "2026-08-25T06:00:00Z" } ``` @@ -277,6 +289,8 @@ curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics - `active_provider_attempts`：attempt ledger 中仍为 `reserved/send_started` 的行数；它是本地数据库 admission 事实，不承诺 Worker 崩溃后的 Provider 幽灵请求已经停止。 - `overdrawn_governance_scopes`：已被实际 usage 超出 reservation 的治理 scope 数，不是超额 attempt 数。 - `total_attempts`、`total_failed_attempts`、`total_dispatches`：Run 表中 lease 取得次数、实际 Run 级失败次数与公平调度 dispatch 次数的总和。 +- `worker_expected_processes` 是部署显式声明的最小进程数；`registered/live/stalled/shortfall` 从未停止的 `worker_processes` generation 与同一 DB UTC cutoff 派生，`last_seen_at == cutoff` 仍为 live。 +- 五个 `worker_last_*` 字段对未停止 generation 取最近（`MAX`）事实；空集合或尚未发生该事件时为 `null`。响应不包含 generation/worker ID，dependency probe 也不会写入或伪造这些时间。 这些都是查询时点的 DB-derived gauges，不是完整事件 counters、历史延迟、审计记录或监控面板；它们绝不能覆盖数据库任务状态。保留期内的有界历史聚合见下一节 `/tasks/history`。 @@ -345,7 +359,21 @@ curl -sS 'http://127.0.0.1:8000/api/v1/tasks/history?window_hours=24' 窗口终点、audit 校验/counter 与三组 Run 延迟都在同一数据库读取快照内完成：PostgreSQL 使用 `REPEATABLE READ READ ONLY`，SQLite 显式开启读事务。因此同一响应不会把不同提交时点的 event 与 Run timestamp 混合；这仍不是跨请求的历史快照或 WORM 保证。每种延迟最多按观测时间、Run ID 的稳定顺序读取最早 10,000 个样本；`sample_count` 是实际参与计算的数量。若还有更多样本，`truncated=true`，分位数只描述这 10,000 个样本，不能当作完整窗口统计。audit counters 不受 latency 样本上限影响。`window_hours` 越界返回 `422`。 -### 3.6 `GET /info` +### 3.6 `GET /metrics/prometheus` + +返回固定 Prometheus text format `0.0.4`： + +```bash +curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus +``` + +成功状态为 `200`，`Content-Type: text/plain; version=0.0.4; charset=utf-8`、`Cache-Control: no-store`、LF 和唯一末尾换行。接口不接受任何 query parameter；存在参数时返回 `422 metrics_query_parameters_not_allowed`。同一 API 进程已有 collection 时立即返回 `429 metrics_scrape_in_progress`，不排队。 + +一次 collection 在同一数据库读快照和同一 DB UTC 下取得 current Run/governance gauges、固定 15 分钟 typed-audit window、固定 1 小时 Run latency 与 Worker aggregate。audit 最多读取 `50,001` 行；第 `50,001` 行使整个请求返回 `503 metrics_observation_limit_exceeded`，不截断 counter。每类 latency 最多读取 `10,001` 个样本，输出前 10,000 个的 p50/p95/p99 与 `truncated=1`。任一 retained audit 的 contract/hash/identity/retention/数值损坏返回 `500 audit_event_integrity_error`；数据库失败返回 `503 metrics_database_unavailable`；renderer 拒绝负数/非有限值并返回 `500 metrics_rendering_error`。所有失败都不返回部分 exposition或异常原文。 + +Redis ping 在数据库快照外，只影响 `llmbenchlab_queue_configured`/`llmbenchlab_queue_available`；Redis down 不会让数据库指标失败。所有 metric type 都是 `gauge`。labels 仅为固定的 `event_type`、latency `phase/quantile`、Worker `state/activity`；不输出 Run、Model、Provider、Worker、Question、policy、URL、hash 或错误文本。JSON `/tasks/metrics` 的 Worker 时间取 active generation 的 `MAX`；Prometheus `llmbenchlab_worker_activity_oldest_age_seconds` 取 `MIN` 后计算最老 age，二者语义不同。完整 family、示例 scrape 和八条规则见 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 与 `deploy/observability/`。 + +### 3.7 `GET /info` 请求： @@ -370,7 +398,7 @@ curl -sS http://127.0.0.1:8000/api/v1/info } ``` -### 3.7 `GET/PUT /governance/policy` +### 3.8 `GET/PUT /governance/policy` 该接口没有认证，只允许可信 loopback 运维调用。`GET` 是安全只读操作：数据库尚无 policy 时返回 `404 governance_policy_not_initialized`，不会顺带创建 policy、scope 或 audit 行。 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 18f5aec..c6318e8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # LLMBenchLab 架构 -本文描述 LLMBenchLab 当前 Phase 1 产品边界、Phase 2 可靠执行与治理工作树，以及可信本地 MMLU-Pro/GPQA-Diamond 真实评测垂直切片。前端仍通过 REST API 操作同一组领域对象，但 API 不执行评测：PostgreSQL/数据库是任务、四层治理、逐 Provider attempt ledger、typed audit 与评分证据的唯一事实来源，Redis Streams 是非权威的 at-least-once 通知层，独立 Worker 或受信本地 CLI 用数据库租约、心跳与 fencing 执行同一 Runner。SQLite 保留为单 Worker 本地兼容路径。Phase 2/3 总状态没有因此完成，这不是公网、HA、生产架构或 SLA 声明。 +本文描述 LLMBenchLab 当前 Phase 1 产品边界、Phase 2 可靠执行/治理/可观测性工作树，以及可信本地 MMLU-Pro/GPQA-Diamond 真实评测垂直切片。前端仍通过 REST API 操作同一组领域对象，但 API 不执行评测：PostgreSQL/数据库是任务、四层治理、逐 Provider attempt ledger、typed audit、Worker progress 与评分证据的唯一事实来源，Redis Streams 是非权威的 at-least-once 通知层，独立 Worker 或受信本地 CLI 用数据库租约、心跳与 fencing 执行同一 Runner。SQLite 保留为单 Worker 本地兼容路径。Exporter、告警和普通文件 archive 都是数据库事实的受控投影，不是第二状态机。Phase 2/3 总状态没有因此完成，这不是公网、HA、生产架构或 SLA 声明。 ## 架构目标与原则 @@ -9,7 +9,7 @@ - **可复现**：Run 保存模型、数据集、Evaluator、Prompt、生成参数、并发和重试策略的快照。 - **失败隔离**：单题失败被持久化并计 0 分，不中断其余题目。 - **秘密最小化**：Web Key 只以 AES-GCM authenticated ciphertext 持久化，部署 keyring 与数据库分离；兼容模式只保存环境变量名。API/Run/Response 不返回 Key 或 envelope，上游成功内容和相关标识在进入持久化边界前检查当前 Key 的精确反射。 -- **事实单一**：Run 状态、取消意图、attempt、租约、Response、聚合、错误、dead-letter、治理 policy/scope/minute bucket、逐调用 reservation/settlement 与 typed audit 只由数据库裁决；Redis、日志、指标和内存不是事实来源。 +- **事实单一**：Run 状态、取消意图、attempt、租约、Response、聚合、错误、dead-letter、治理 policy/scope/minute bucket、逐调用 reservation/settlement、typed audit 与 Worker progress 只由数据库裁决；Redis、日志、Prometheus、告警、archive manifest 和内存不是第二事实来源。 - **at-least-once 与幂等**：通知可重复，逐题写入以 `(run_id, question_id)` 唯一并由当前 lease token fencing；不声称 Provider 调用或计费 exactly-once。 - **本地 admission，不冒充账单**：managed Web/API Run 在数据库中执行 global/provider/model/run 四层限流、预算与背压；它限制本地开始新 attempt 的许可，不会取消崩溃后仍在 Provider 运行的幽灵请求，也不把本地保守 consumed 伪装成 Provider 账单真值。 - **边界稳定**：Adapter、Evaluator、Dataset Loader 和 Runner 通过明确接口解耦，便于后续替换实现。 @@ -554,13 +554,13 @@ React 主导航包含独立的 Runs 列表页。该页通过 `GET /runs` 以 20 数据库平台迁移是 stopped-source/offline-empty-target 操作，不是双写或在线复制： 1. 停止 SQLite 源的 API/Worker 和新 Run 创建，确认无 `pending/running` Run。导入器使用 read-only URI，校验 integrity、foreign keys 和 Alembic head，不修改源文件。 -2. PostgreSQL 目标必须已在 head，12 张核心/治理表必须为空且不可对外服务；源库不得含 active reservation，并必须在连接目标前用全部 reservation ledger 重算每个 scope/minute 物化值，任何高、低漂移或缺 bucket 都拒绝导入。事务级 advisory lock 在空库检查之前串行化竞争导入，随后对 `alembic_version` 和核心表取 `ACCESS EXCLUSIVE` 锁。 -3. `governance_policies → models → model_credentials → benchmarks → questions → governance_scopes → evaluation_runs → evaluation_responses → governance_minute_buckets → question_executions → provider_call_reservations → audit_events` 在一个目标事务中按依赖顺序复制，包括 encrypted credential envelope、ledger/audit 与 typed Provider metadata，但不解密或输出秘密列值。源、precommit target 都只输出 row count、主键集 digest 和 canonical row digest；任一 precommit 失配/复制失败都整体 rollback，CLI 退出 `2`。 +2. PostgreSQL 目标必须已在 head，13 张核心/治理表必须为空且不可对外服务；源库不得含 active reservation 或按同一 DB UTC cutoff 仍 live 的 Worker generation，并必须在连接目标前用全部 reservation ledger 重算每个 scope/minute 物化值，任何高、低漂移或缺 bucket 都拒绝导入。事务级 advisory lock 在空库检查之前串行化竞争导入，随后对 `alembic_version` 和核心表取 `ACCESS EXCLUSIVE` 锁。 +3. `governance_policies → models → model_credentials → benchmarks → questions → governance_scopes → evaluation_runs → evaluation_responses → governance_minute_buckets → question_executions → provider_call_reservations → audit_events → worker_processes` 在一个目标事务中按依赖顺序复制，包括 encrypted credential envelope、ledger/audit、stopped/stale Worker facts 与 typed Provider metadata，但不解密或输出秘密列值。源、precommit target 都只输出 row count、主键集 digest 和 canonical row digest；任一 precommit 失配/复制失败都整体 rollback，CLI 退出 `2`。 4. COMMIT 确认后，工具在独立的只读 `REPEATABLE READ` 事务中取稳定 postcommit snapshot，再对账并输出第三组摘要。全部完成才退出 `0`。 带凭据的目标 DSN 必须通过 `--target-env` 从受控环境读取；`--target` 拒绝 URL password 和 password query。COMMIT 未获得 PostgreSQL 确认时退出 `4`/`commit_outcome_unknown`；由于事务原子性，目标可能为空，也可能是完整的 precommit 快照。COMMIT 已确认但连接收尾、postcommit 快照/对账或报告失败时退出 `3`/`committed_but_verification_failed`；这时目标已提交完整 precommit 快照，不会自动回滚。两种结果都禁止盲目重试，必须保持目标离线，按已输出摘要独立检查目标是空还是完整提交。非空目标会拒绝再次导入，工具也不提供 PostgreSQL 到 SQLite 的反向同步。 -Alembic `20260827_0004` 引入上述治理/审计表与 Run/Response 字段。downgrade guard 会在第一条 DDL 前拒绝任何可能丢失的 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 事实；正常 bootstrap 后的数据库因此不能把 `0004 → 0003` 当普通代码回滚。隔离空数据库可验证 `0004 → 0003 → 0004`，已使用环境应优先向前修复，或恢复经核验的 pre-0004 备份并单独保留 post-0004 ledger/audit 归档。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 +Alembic `20260827_0004` 引入治理/审计表与 Run/Response 字段；`20260828_0005` 增加 `worker_processes` 与 audit retention/exporter 扫描索引。两个 downgrade guard 都在第一条 DDL 前拒绝可能丢失的事实：0005 拒绝任意 Worker generation，0004 拒绝 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 证据。隔离空数据库分别验证 `0005 ↔ 0004` 与 `0004 ↔ 0003`；已使用环境应优先向前修复，或恢复经核验的旧备份并单独保留新 schema 证据。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 ## 错误处理与可观察性 @@ -581,12 +581,12 @@ Alembic `20260827_0004` 引入上述治理/审计表与 Run/Response 字段。do 可观测基础包含： - API 忽略客户端传入的 `X-Request-ID`，为每个请求始终生成全新的 server-side UUID 并在响应中回传。CORS 请求 header allowlist 不含 `X-Request-ID`，但会向浏览器暴露响应中的该 header，避免客户端把 write-only Key 复制到 correlation ID 后迫使服务反射或记录。新 Run 使用 run ID 作为稳定 correlation ID，通知、Worker、Runner 和 Question 事件继承该链路。 -- LLMBenchLab **应用 logger** 使用字段白名单的 JSON formatter，记录 request route template、run/question/worker、attempt、lease token、message ID 和结果；不记录请求体、header、原始回答或异常文本，只可记异常类型。这一保证不涵盖所有 Uvicorn/access log handler，所以秘密不得出现在 URL、query 或 path。持久 typed audit 与 logger 分离，前者是应用 append-only、event-key 幂等且受 retention 管理的历史事实，但仍不是不可篡改/WORM 日志。 +- LLMBenchLab 配置的已登记应用 logger 与进程内 Uvicorn/SQLAlchemy client logger 统一通过 JSON sanitizer；消息必须是 literal，structured extra 先按字段白名单，再按固定 enum、canonical UUID/Redis stream ID 和有限数值合同规范化，非法 ID 省略，未知字符串只投影固定 `unsupported`。Redis Run 通知在进入 Worker 前同样要求 Run/correlation canonical UUID；Uvicorn access log 默认关闭。Alembic migration CLI 使用独立 `fileConfig`/console 输出，不经过应用 sanitizer；该边界也不治理反向代理、PostgreSQL、Redis server、Docker daemon 或崩溃转储，所以秘密仍不得出现在 URL、query、path、argv 或迁移诊断。持久 typed audit 与 logger 分离，前者是应用 append-only、event-key 幂等且受 retention 管理的历史事实，但仍不是不可篡改/WORM 日志。 - `/live` 是纯进程存活检查，不访问 DB/Redis/Provider；`/health` 保持 DB-only 兼容语义；`/ready` 并行检查 DB 连接/Alembic head 与 Redis，不探测 Provider。Redis 失败时返回 `503/degraded`，但 DB/head 可用时 `accepting_runs=true` 且对账可用；DB/head 失败时 `not_ready` 且不接收任务。 - `/ready` 将同步 DB 探测放入 `asyncio.to_thread` 并限制 HTTP 等待时间。async timeout 不会取消已进入线程的驱动调用；后台资源的真正上界仍依赖数据库 driver/connect/pool timeout。 -- `/tasks/metrics` 通过三组有界 DB 聚合查询派生任务、governance backlog/delay/exhaustion、active Provider attempt、overdrawn scope、attempt/failure/dispatch 等当前 gauges；它不是单次查询，也不是历史 counter。`/tasks/history?window_hours=1..2160` 在一个 PostgreSQL `REPEATABLE READ READ ONLY`/显式 SQLite 读快照内确定 DB UTC 窗口，逐条验证 retained audit 的 contract/hash/identity/retention 后聚合任务 counters，再从同一快照的 Run 时间字段计算 queue/execution/end-to-end p50/p95/p99；损坏 audit 使整个响应 fail closed，不返回部分结果。pending/running cancel 都写 `run_cancel_requested`，dead-letter 使用专用 `run_dead_lettered` counter 而不混入一般 terminal 原因。每组 latency 最多读取 10,000 个样本并显式返回 `truncated`。`/runs/{id}/audit` 按 `(occurred_at,id)` 稳定分页并在读取时复核 allowlist/payload hash。 -- 每题 Provider request ID、返回模型名、system fingerprint、finish reason 和 HTTP attempt count 已作为安全归一化 typed 字段写入 EvaluationResponse、读取 API 与报告。凭据变更/拒绝/解密失败另写 security-retention audit。可信本地初次 canary 仍只固化在 Run snapshot，`resume` canary 尚无独立追加事件；也没有 Prometheus exporter、自动告警、tracing 或管理员不可篡改证明。 -- Worker 容器探针检查 DB/head 和 Redis 能力。DB/head 失败退出 1；Redis 失败输出 degraded 但退出 0，因为 DB reconciliation 仍可用。它是 dependency/capability readiness，不是 Worker 主循环或 event-loop liveness。 +- `/tasks/metrics` 派生任务/governance 与匿名 Worker expected/registered/live/stalled/shortfall、最近聚合进展；`/tasks/history?window_hours=1..2160` 在一个 PostgreSQL `REPEATABLE READ READ ONLY`/显式 SQLite 读快照内逐条验证 retained audit 后聚合任务 counters 与 Run p50/p95/p99。`/metrics/prometheus` 复用同一 collector，在固定 15 分钟 audit/1 小时 latency window、hard row cap 和 per-process single-flight 下输出全部 gauge 的固定低基数 exposition。Redis availability 是快照外观察，不改变 DB 事实。损坏 audit 使 history/exporter 整体 fail closed，不返回部分结果；`/runs/{id}/audit` 仍按 `(occurred_at,id)` 稳定分页。 +- 每个长运行 Worker 注册唯一 generation；scan/claim/progress/lease-heartbeat 只置固定 bit，由 event-loop recorder 每 5 秒最多一笔短事务用同一 DB UTC 合并。无事件零写，故 timer 不能在主循环卡死后制造 keepalive；crash generation 保持 active 后自然派生为 stale。JSON 公开 active generation 时间的 `MAX`，Prometheus oldest age 使用 `MIN`，两者都不输出 Worker/generation ID。dependency probe 明确 `main_loop_progress=not_checked`。 +- 每题 Provider metadata 已安全归一化写入 Response/API/报告；凭据变更/拒绝/解密失败另写 security-retention audit。`llmbenchlab-audit-retention` 提供权限收紧的 canonical JSONL、完全离线 verify、精确 reconcile/restore/delete；archive 保存完整内部事实，hash 不是签名/WORM。仓库交付八条 Prometheus 规则与 Runbook，但不部署告警发送器、Dashboard、tracing 或管理员不可篡改存储。 ## 部署拓扑与安全边界 @@ -600,13 +600,13 @@ Alembic `20260827_0004` 引入上述治理/审计表与 Run/Response 字段。do v2 aggregate schema 为 `llmbenchlab-phase2-slo-evidence-v2`。aggregate 与 raw child evidence 都位于 Git 忽略目录；aggregate 只含 commit/source hash、稳定指纹、child 相对路径/hash、匿名参与计数、脱敏统计/判定、容量模型和 cleanup 摘要的 allowlist，不复制 raw identity、stdout/log、DSN/URL、环境变量、题目、Prompt/Response 或秘密材料。公开状态可以给出 Git 忽略的外层 aggregate 相对路径和内容 SHA；raw child、aggregate 内嵌 child 路径与环境/配置明细仍是内部证据，只摘录人工复核的 commit 与匿名结果。child 使用独立进程组，超时/中断后给 scoped cleanup 420 秒。 -历史 v1 在 clean `dfa67abb1a9a0418a7e3337c179f816e3c69f121` 上为 15/18，保持 `unqualified`。v2 已在 clean `b6a35fef1dd069ebb54b69955058915c722aa34d` 上从零完成 1+5，23/23 项 SLO、每轮 hard invariant 和 cleanup 全部通过，aggregate 内容 SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；同一实现 SHA 的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 必需 job 成功。该架构把正确性硬门禁、统计资格和证据最小化分层；结果只描述被记录的固定单机 Mock 硬件、配置和 commit，不是 Provider 性能、生产 SLO/SLA、HA、无限扩展或 Phase 2 整体完成证明。GitHub-hosted CI 只验证 validator、统计和失败路径，不承担绝对性能门禁。基线解释见 [`PERFORMANCE.md`](./PERFORMANCE.md)，扩缩容、预算/背压、settlement unknown、DB/Redis/lease 恢复与 0004 安全回滚见 [`OPERATIONS.md`](./OPERATIONS.md)。 +历史 v1 在 clean `dfa67abb1a9a0418a7e3337c179f816e3c69f121` 上为 15/18，保持 `unqualified`。v2 已在 clean `b6a35fef1dd069ebb54b69955058915c722aa34d` 上从零完成 1+5，23/23 项 SLO、每轮 hard invariant 和 cleanup 全部通过，aggregate 内容 SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；同一实现 SHA 的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 必需 job 成功。该架构把正确性硬门禁、统计资格和证据最小化分层；结果只描述被记录的固定单机 Mock 硬件、配置和 commit，不是 Provider 性能、生产 SLO/SLA、HA、无限扩展或 Phase 2 整体完成证明。GitHub-hosted CI 只验证 validator、统计和失败路径，不承担绝对性能门禁。基线解释见 [`PERFORMANCE.md`](./PERFORMANCE.md)，扩缩容、预算/背压、settlement unknown、DB/Redis/lease 恢复与 0005/0004 安全回滚见 [`OPERATIONS.md`](./OPERATIONS.md)。 ## 当前限制 - PostgreSQL 租约已支持受限多 Worker 协调；SQLite 仍只适合单 Worker、单机低并发。这不是无限水平扩展或 HA 保证。 - managed Web/API Run 已实现数据库权威四层限流/预算/背压与公平调度；默认 policy 可关闭限制，`legacy_unmanaged` CLI 不覆盖，当前 global scope 锁会串行化新 admission，Mock 容量基线也不是生产调优结论。 -- typed audit/history 与逐题 Provider metadata 已实现，但 append-only 不是 WORM；尚无 exporter、自动告警、tracing/监控面板、`resume` canary 独立事件或公网对象级访问控制。 +- typed audit/history/archive、低基数 exporter、八条规则、Worker DB-time progress 与逐题 Provider metadata 已实现，但 append-only/普通文件 hash 不是 WORM；尚无告警发送器、tracing/认证监控面板、`resume` canary 独立事件或公网对象级访问控制。 - at-least-once 不能防止 `send_started` 后崩溃留下 Provider 幽灵请求，或 Provider 响应到本地 COMMIT 之间的重复远程调用/费用；本地 ledger 幂等和保守结算都不是远端 exactly-once/账单真值。 - 上游是否真正遵守 `seed`、temperature 等参数由供应商决定；同配置不保证逐 token 完全确定。 - 仅支持客观的 exact match、multiple choice、numeric；不执行代码，不提供 LLM Judge、Arena、Agent 或长上下文专用协议。 @@ -618,7 +618,7 @@ v2 aggregate schema 为 `llmbenchlab-phase2-slo-evidence-v2`。aggregate 与 raw ## 后续扩展方式 -1. **生产治理与观测收口**：在已有 DB governance、attempt ledger、typed audit/history 与 Mock 容量基线上增加认证/租户 policy、exporter/告警/tracing、审计归档/WORM 选项，并在目标硬件与精确提交上重新校准容量。 +1. **生产治理与观测收口**：在已有 DB governance、attempt ledger、typed audit/archive、Worker progress、exporter/规则与 Mock 容量基线上增加认证/租户 policy、告警发送/认证 Dashboard/tracing、签名/WORM 审计选项，并在目标硬件与精确提交上重新校准容量。 2. **数据集插件**：在已有固定 MMLU-Pro/GPQA 转换器上增加通用注册、签名、分片和更多 dataset plugin；继续把原始来源版本与转换器版本纳入 Hash 元数据。 3. **Evaluator 插件**：以版本化 Registry 增加 IFEval、代码沙箱、LLM Judge 和 Pairwise Judge；任何评分语义变化必须升级 protocol version。 4. **Adapter 扩展**：实现新的供应商 Adapter，而不是在 Runner 中添加条件分支；能力声明用于标识 seed、usage、工具调用和上下文窗口支持。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 00e192c..641a52d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -8,7 +8,7 @@ LLMBenchLab 当前有三条本地运行路径： - Phase 2 Compose 可靠执行路径：PostgreSQL 是唯一任务事实来源，Redis Streams 是 at-least-once 通知层，API 与 Worker 是独立进程，migrate 是唯一 Alembic upgrade owner，Nginx 提供前端。 - 可信本地正式评测 CLI：直接连接已迁移数据库，固定下载/转换 MMLU-Pro 或 GPQA-Diamond，在同一进程做 Provider preflight、运行有界 Runner、恢复缺失题并导出全量报告；不要求浏览器、API、Redis 或常驻 Worker。 -可靠执行基础已经覆盖租约、心跳、fencing、幂等 Response、取消、有限重试、数据库 reconciliation 和故障恢复；数据库权威的 global/provider/model/run 治理、逐 HTTP attempt ledger、背压、公平 slice、typed audit、历史延迟及 Mock 容量基线也已进入 Phase 2 工作树，`llmbenchlab-protocol-v1` 评分含义没有改变。部署仍不等于生产高可用；容量边界见 [`PERFORMANCE.md`](PERFORMANCE.md)，告警、故障和 0004 安全回滚见 [`OPERATIONS.md`](OPERATIONS.md)。 +可靠执行基础已经覆盖租约、心跳、fencing、幂等 Response、取消、有限重试、数据库 reconciliation 和故障恢复；数据库权威的 global/provider/model/run 治理、逐 HTTP attempt ledger、背压、公平 slice、typed audit/archive、Worker progress、固定 exporter/规则、历史延迟及 Mock 容量基线也已进入 Phase 2 工作树，`llmbenchlab-protocol-v1` 评分含义没有改变。部署仍不等于生产高可用；容量边界见 [`PERFORMANCE.md`](PERFORMANCE.md)，告警、故障、retention 与 0005/0004 安全回滚见 [`OPERATIONS.md`](OPERATIONS.md)。 > Compose 是本地故障验证配置，不是生产方案。它没有认证、授权、TLS、正式 secret manager、自动备份/PITR、告警或 HA；不要直接暴露公网，也不要把示例密码当作生产秘密。 @@ -238,6 +238,10 @@ Pydantic 应用设置优先读取 `LLMBENCHLAB_*`，并为数据库、Redis、CO | `LLMBENCHLAB_WORKER_RETRY_BACKOFF_BASE_SECONDS` | `1` | 重试退避基数 | | `LLMBENCHLAB_WORKER_RETRY_BACKOFF_CAP_SECONDS` | `30` | 重试退避上限 | | `LLMBENCHLAB_WORKER_SHUTDOWN_GRACE_SECONDS` | `30` | SIGTERM 后等待活动 Run 的应用 grace | +| `LLMBENCHLAB_WORKER_PROGRESS_FLUSH_SECONDS` | `5` | 真实 scan/claim/progress/heartbeat bit 合并写入 DB 的最大间隔；timer 不生成 keepalive | +| `LLMBENCHLAB_WORKER_PROGRESS_STALE_SECONDS` | `60` | DB UTC 下 active generation 的 stale cutoff | +| `LLMBENCHLAB_WORKER_EXPECTED_PROCESSES` | `1` | 部署明确声明的最小 live Worker 数；扩缩容必须同步修改 | +| `LLMBENCHLAB_WORKER_RECOVERY_ALERT_SECONDS` | `60` | expired lease age 告警比较阈值 | | `LLMBENCHLAB_MOCK_GENERATION_DELAY_SECONDS` | `0` | 只用于确定性 Mock 故障测试；不改变报告 latency 或协议评分 | | `LLMBENCHLAB_CREDENTIAL_KEYS_FILE` | 仓库根 `.secrets/credential-keys.json` | API 加密、Worker 解密 Web Key 的共享 keyring；空值会关闭 stored 模式并 fail closed | @@ -270,6 +274,10 @@ Pydantic 应用设置优先读取 `LLMBENCHLAB_*`，并为数据库、Redis、CO | `LLMBENCHLAB_COMPOSE_WORKER_RETRY_BACKOFF_BASE_SECONDS` | `1` | 映射到 Worker retry backoff base | | `LLMBENCHLAB_COMPOSE_WORKER_RETRY_BACKOFF_CAP_SECONDS` | `30` | 映射到 Worker retry backoff cap | | `LLMBENCHLAB_COMPOSE_WORKER_SHUTDOWN_GRACE_SECONDS` | `30` | 应用 grace；容器另有 45 秒 stop grace | +| `LLMBENCHLAB_COMPOSE_WORKER_PROGRESS_FLUSH_SECONDS` | `5` | 映射到 Worker progress flush | +| `LLMBENCHLAB_COMPOSE_WORKER_PROGRESS_STALE_SECONDS` | `60` | 映射到 Worker stale cutoff | +| `LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES` | `1` | Compose 目标 Worker 数；`--scale` 时必须同步设置 | +| `LLMBENCHLAB_COMPOSE_WORKER_RECOVERY_ALERT_SECONDS` | `60` | exporter lease recovery 告警阈值 | | `LLMBENCHLAB_COMPOSE_REDIS_BLOCK_MILLISECONDS` | `1000` | 映射到 Worker Redis blocking read 上限 | | `LLMBENCHLAB_COMPOSE_REDIS_OPERATION_TIMEOUT_SECONDS` | `1` | 容器 Redis 操作 timeout | | `LLMBENCHLAB_COMPOSE_DATABASE_POOL_SIZE` | `5` | 映射到 API/Worker 数据库 pool size | @@ -334,7 +342,7 @@ python -m app.db.prepare_migrations && alembic upgrade head && alembic check `api` 与 `worker` 必须等待 migrate exit 0，然后仅执行 head check。不要把 Alembic 命令加回 API/Worker entrypoint，也不要同时运行多个 migration owner。 -`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时于第一条 DDL 前拒绝；正常使用后的数据库不能把它当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会在 DDL 前拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 +`0005 -> 0004` 在 `worker_processes` 有任意 generation fact 时于第一条 DDL 前拒绝；`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时同样拒绝。正常使用后的数据库不能把它们当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0005/0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 ### 5.3 备份与恢复证据边界 @@ -352,7 +360,7 @@ Compose 定义六个 service，其中五个常驻，`migrate` 为一次性任务 | `redis` | Redis 7 Streams 通知层，AOF everysec | `redis-cli ping`；`redis-data`；无 host port | | `migrate` | 唯一 Alembic preflight/upgrade/check owner | 等 PostgreSQL healthy；成功后 exit 0，不常驻 | | `api` | FastAPI CRUD、write-only Key 加密、Run commit 与 best-effort publish | 等 migrate 成功；只读挂载 keyring；启动只 head check；`/ready` 为容器 health；loopback API port | -| `worker` | 独立租约 Worker、stored Key 解密、DB reconciliation、Redis consume/ACK | 等 migrate 成功；只读挂载同一 keyring；启动只 head check；dependency probe；容器 stop grace 45 秒 | +| `worker` | 独立租约 Worker、stored Key 解密、DB reconciliation、Redis consume/ACK、DB-time process progress | 等 migrate 成功；只读挂载同一 keyring；启动只 head check；dependency-only probe；容器 stop grace 45 秒 | | `frontend` | Nginx 静态站与 `/api/` 同源代理 | 等 API healthy；loopback frontend port；关闭 API request buffering，避免 Key body 被 Nginx 临时落盘 | ### 6.1 启动、检查和停止 @@ -372,6 +380,7 @@ curl -sS http://127.0.0.1:8000/api/v1/live curl -sS http://127.0.0.1:8000/api/v1/health curl -sS http://127.0.0.1:8000/api/v1/ready curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics +curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus curl -sS http://127.0.0.1:8080/healthz ``` @@ -425,14 +434,14 @@ raw child 与 aggregate evidence 都保留在 Git 忽略的 `.pytest_cache/artif ## 7. SQLite→PostgreSQL 单向导入 runbook -导入器是显式、一次性的 12 表复制，不会自动在应用启动时运行，也不会反向同步。`model_credentials` 含认证密文，ledger/audit 又含运行与费用元数据；工具输出仍只包含行数与摘要，但源、目标和数据库备份必须作为敏感数据保护。 +导入器是显式、一次性的 13 表复制，不会自动在应用启动时运行，也不会反向同步。`model_credentials` 含认证密文，ledger/audit/worker progress 又含运行、费用和故障时间元数据；工具输出仍只包含行数与摘要，但源、目标和数据库备份必须作为敏感数据保护。 ### 7.1 前置条件 1. 停止源 SQLite 的 API、Worker 和所有写进程；同时停止目标 API/Worker 和用户入口。导入器只能保证自身连接只读，不能证明外部写进程已停止。 2. 按组织要求创建并独立验证源与目标备份。仓库没有生产备份/恢复演练，不得把本 runbook 描述为已验证灾难恢复。 -3. 源必须是文件型 SQLite、处于当前 Alembic head；不能有 `pending` 或 `running` Run，也不能有 `reserved/send_started` attempt。导入器会在打开目标前用全部 never-delete ledger 重算所有 scope 与 minute-bucket 的 active/reserved/consumed/overdrawn 事实；物化值任何高、低漂移、scope 类型错配或缺 bucket 都在接触目标前拒绝。 -4. 目标必须是当前 head 的 PostgreSQL，12 张核心/治理表必须为空。先由唯一 migrate owner 完成 schema，再保持 API/Worker 停止。 +3. 源必须是文件型 SQLite、处于当前 Alembic head；不能有 `pending` 或 `running` Run、`reserved/send_started` attempt，或按同一 DB UTC cutoff 仍 live 的 Worker generation。stopped/stale generation 会精确复制。导入器会在打开目标前用全部 never-delete ledger 重算所有 scope 与 minute-bucket 的 active/reserved/consumed/overdrawn 事实；物化值任何高、低漂移、scope 类型错配或缺 bucket 都在接触目标前拒绝。 +4. 目标必须是当前 head 的 PostgreSQL，13 张核心/治理表必须为空。先由唯一 migrate owner 完成 schema，再保持 API/Worker 停止。 5. 在受信环境运行；源可能包含题目、参考答案、原始模型输出和错误内容。保护源文件、终端输出和摘要日志。 导入器以 SQLite URI `mode=ro` 打开源、设置 `PRAGMA query_only=ON`，在显式读事务内执行 `integrity_check`、`foreign_key_check`、head/no-active 检查与 snapshot。任何一项失败都会在接触目标数据前停止。 @@ -487,12 +496,12 @@ docker compose run --rm \ 目标流程在一个 PostgreSQL 事务内： 1. 获取固定 transaction advisory lock，串行化两个 importer。 -2. 检查 head，随后对 `alembic_version` 与 12 张核心/治理表获取 `ACCESS EXCLUSIVE` lock，再次检查 head 与空表。 -3. 按依赖顺序复制 `governance_policies`、`models`、`model_credentials`、`benchmarks`、`questions`、`governance_scopes`、`evaluation_runs`、`evaluation_responses`、`governance_minute_buckets`、`question_executions`、`provider_call_reservations`、`audit_events`。 +2. 检查 head，随后对 `alembic_version` 与 13 张核心/治理表获取 `ACCESS EXCLUSIVE` lock，再次检查 head 与空表。 +3. 按依赖顺序复制 `governance_policies`、`models`、`model_credentials`、`benchmarks`、`questions`、`governance_scopes`、`evaluation_runs`、`evaluation_responses`、`governance_minute_buckets`、`question_executions`、`provider_call_reservations`、`audit_events`、`worker_processes`。 4. 提交前比较行数、主键集合 SHA-256 与 canonical row SHA-256；失败整体 rollback。 5. COMMIT 成功确认后，在单个 `REPEATABLE READ`、`READ ONLY` 事务中做 post-commit snapshot。 -成功输出三组、每组 12 行的 content-free 摘要：`phase=source`、`phase=precommit_target`、`phase=postcommit_target`。每张表的 `row_count`、`pk_set_digest` 和 `canonical_row_digest` 必须三阶段一致。摘要不打印行内容、密文或 URL，但行数/hash 仍是敏感运维元数据。导入后还必须把与源库匹配的 keyring 通过目标环境的 secret 流程单独交付给 API/Worker；导入器不会复制 keyring 文件。 +成功输出三组、每组 13 行的 content-free 摘要：`phase=source`、`phase=precommit_target`、`phase=postcommit_target`。每张表的 `row_count`、`pk_set_digest` 和 `canonical_row_digest` 必须三阶段一致。摘要不打印行内容、密文或 URL，但行数/hash 仍是敏感运维元数据。导入后还必须把与源库匹配的 keyring 通过目标环境的 secret 流程单独交付给 API/Worker；导入器不会复制 keyring 文件。 ### 7.5 Exit code 与恢复动作 @@ -500,10 +509,10 @@ docker compose run --rm \ | --- | --- | --- | --- | | `0` | completed and reconciled | COMMIT 已确认，post-commit 摘要匹配 | 保存脱敏摘要；检查 head/ready 后再启动 API/Worker | | `2` | pre-commit failure | preflight/copy/提交前对账失败；目标事务若已开始会 rollback | 保留错误与源；确认目标仍为空并修复原因后，才考虑重新执行 | -| `4` | `commit_outcome_unknown` | PostgreSQL 未确认 COMMIT；原子事务意味着目标可能为空，也可能已完整提交 | 立即停止；保持应用停机，检查目标 12 表和摘要；禁止盲目重试、truncate 或覆盖 | +| `4` | `commit_outcome_unknown` | PostgreSQL 未确认 COMMIT；原子事务意味着目标可能为空，也可能已完整提交 | 立即停止；保持应用停机，检查目标 13 表和摘要；禁止盲目重试、truncate 或覆盖 | | `3` | `committed_but_verification_failed` | COMMIT 已确认，但 post-commit snapshot/比较或摘要输出未完成 | 将目标视为已提交；保持只读检查并补做对账；禁止重新导入或清空 | -exit 4 时，只有在独立检查证明 12 表仍为空后才可按新变更重新运行；若非空，按“可能已完整提交”保护现场。exit 3 已明确提交，不得把它当成 rollback。任何不确定状态都应升级给数据库负责人，而不是靠重复命令猜测。 +exit 4 时，只有在独立检查证明 13 表仍为空后才可按新变更重新运行；若非空，按“可能已完整提交”保护现场。exit 3 已明确提交，不得把它当成 rollback。任何不确定状态都应升级给数据库负责人，而不是靠重复命令猜测。 ### 7.6 导入后与回退 @@ -538,21 +547,25 @@ Compose API healthcheck 使用 `/ready`，所以 Redis 停止时容器会显示 docker compose exec worker python -m app.worker_probe ``` -probe 检查数据库连接、Alembic head 与队列能力：DB/head 或队列配置错误 exit 1；Redis 运行时不可用但 DB reconciliation 可用时输出 `degraded` 且 exit 0。它是依赖 capability/readiness probe，不观察 Worker 主循环、当前 lease heartbeat、事件循环卡死或执行吞吐，不能称为完整 Worker liveness。 +probe 检查数据库连接、Alembic head 与队列能力：DB/head 或队列配置错误 exit 1；Redis 运行时不可用但 DB reconciliation 可用时输出 `degraded` 且 exit 0。输出明确包含 `probe_scope=dependencies_only`、`main_loop_progress=not_checked`。它不写 progress，也不观察当前 Worker 主循环、lease heartbeat、事件循环卡死或执行吞吐；主循环 liveness 必须使用数据库 `worker_processes` 聚合与 shortfall 告警。 ### 8.3 结构化日志 -LLMBenchLab 应用 logger 输出单行脱敏 JSON，包含 allowlist event、request/correlation ID、run/question、worker、attempt、lease token、message ID、结果和异常类型。它不应记录 Authorization、DSN/Redis URL、请求正文、Provider 请求/响应正文、完整题目或原始模型输出。API 始终生成服务端 UUID request ID，不信任或回显客户端 `X-Request-ID`；Run correlation 默认稳定使用 Run ID。 +LLMBenchLab 应用 logger 输出单行脱敏 JSON，包含 allowlist event、request/correlation ID、run/question、worker、attempt、lease token、message ID、结果和固定的异常存在标记 `exception_type="suppressed"`；它不反射具体异常类名。它不应记录 Authorization、DSN/Redis URL、请求正文、Provider 请求/响应正文、完整题目或原始模型输出。API 始终生成服务端 UUID request ID，不信任或回显客户端 `X-Request-ID`；Run correlation 默认稳定使用 Run ID。 -这个保证只覆盖 LLMBenchLab 配置的应用 logger。Uvicorn/error/access handler 仍可能使用原生日志格式；不得把当前实现描述为所有容器日志统一 JSON。秘密也不能放入 URL，因为反向代理/access logger 可能记录 URL。 +应用配置同时治理 Uvicorn、SQLAlchemy 等已知进程内 Python logger：消息/异常只通过固定 allowlist 投影，Uvicorn access log 默认关闭，未知 extra 不进入 JSON。Alembic migration CLI 使用独立 `fileConfig`/console 输出，不经过应用 sanitizer；该边界也不能约束反向代理、PostgreSQL、Redis、Docker daemon、驱动原生日志或崩溃转储，因此秘密绝不能放入 URL、argv、迁移输出或环境诊断。新增 logger 调用必须使用 literal message，不能把 exception/响应/请求对象插值进 message。 ### 8.4 Task metrics -`/api/v1/tasks/metrics` 从 PostgreSQL 当前行派生：pending/due/running/expired/cancel/retry/dead-letter/queue error，以及 managed backlog、governance delayed/exhausted、active Provider attempts、overdrawn scopes、lease/failed attempt 和 dispatch 累计。它是只读 gauges，不参与调度、不覆盖数据库状态。 +`/api/v1/tasks/metrics` 从 PostgreSQL 当前行派生：pending/due/running/expired/cancel/retry/dead-letter/queue error，managed backlog/governance/attempt gauges，以及 Worker expected/registered/live/stalled/shortfall 与最近聚合进展时间。它是只读 gauges，不参与调度、不覆盖数据库状态。 `/api/v1/tasks/history?window_hours=1..2160` 在同一 PostgreSQL `REPEATABLE READ READ ONLY`/显式 SQLite 读快照中确定 DB UTC 窗口，逐条验证 retained audit 的 contract/hash/identity/retention 后聚合 counter，再从同一快照的 Run 时间字段计算 queue/execution/end-to-end p50/p95/p99。损坏 audit 使整个请求返回 `500 audit_event_integrity_error`，不提供部分数据；每类延迟最多 10,000 个样本并显式报告 `truncated`。`/api/v1/runs/{run_id}/audit` 提供稳定分页的应用 append-only audit；它不是 WORM 或数据库管理员防篡改证明。 -当前仍没有 Prometheus exporter、统一第三方日志/trace、告警发送器或生产 SLO。operational/security audit 分别至少保留 90/365 天，清理必须先归档/rollup 且不在请求链路执行。具体轮询、症状告警和 settlement 对账见 [OPERATIONS.md](OPERATIONS.md)。 +`/api/v1/metrics/prometheus` 提供固定低基数、全部 gauge 的 text exposition；每进程单次 collection、15 分钟 audit hard cap 和 1 小时 latency cap 用于约束抓取压力。示例 scrape/八条规则在 `deploy/observability/`，仓库不部署 Prometheus、Alertmanager、通知发送器或生产 SLO。具体阈值与逐条响应见 [OPERATIONS.md](OPERATIONS.md)。 + +### 8.5 Audit retention maintenance + +operational/security audit 分别至少保留 90/365 天，清理不在请求链路执行。使用 `llmbenchlab-audit-retention` 的 `archive/verify/reconcile/restore/delete`；输出目录必须由当前维护用户拥有且不可 group/other 写，建议 `0700`，archive 文件固定 `0600`。`verify` 完全离线；mutation 必须在 API/Worker/audit writers 停止的维护窗口使用 archive 精确 SHA-256，并在 exit `3/4` 时先只读 reconcile、禁止盲目重试。完整命令与失败语义见 [OPERATIONS.md](OPERATIONS.md)。 ## 9. 升级、重启与回滚 @@ -580,7 +593,7 @@ LLMBenchLab 应用 logger 输出单行脱敏 JSON，包含 allowlist event、req ### 9.3 Schema/code 回滚 -代码回滚必须与当前 schema、API 和 `protocol_version` 兼容。回退 `0004` 前必须停止 admission/API/Worker、对账 active reservation，并归档/核验全部治理、ledger、audit 和 Provider metadata；任一证据存在时 downgrade 会在 DDL 前拒绝。优先向前修复或恢复经验证的 pre-0004 备份，同时单独保留 post-0004 账本/审计归档。回退 `0003` 前还必须确认 `model_credentials` 为空；回退 `0002` 前必须确认没有 `pending/running`。详见 [OPERATIONS.md](OPERATIONS.md)。 +代码回滚必须与当前 schema、API 和 `protocol_version` 兼容。回退 `0005` 前必须停止 Worker并处理/保存 `worker_processes` facts；任一 generation 行都会在 DDL 前拒绝。回退 `0004` 前还必须停止 admission/API/Worker、对账 active reservation，并归档/核验全部治理、ledger、audit 和 Provider metadata；任一证据存在时同样拒绝。优先向前修复或恢复经验证的旧备份，同时单独保留新 schema 证据。回退 `0003` 前还必须确认 `model_credentials` 为空；回退 `0002` 前必须确认没有 `pending/running`。详见 [OPERATIONS.md](OPERATIONS.md)。 不同 protocol version、Benchmark version 或 dataset hash 不能因回滚无提示混排。优先使用经过平台验证的完整备份恢复，而不是盲目 downgrade；本仓库当前没有 PostgreSQL→SQLite 自动回退或生产恢复演练。 @@ -592,11 +605,11 @@ LLMBenchLab 应用 logger 输出单行脱敏 JSON，包含 allowlist event、req | 网络 | API/frontend loopback；PG/Redis 内部 Compose 网络；Provider Chat 真 SSE 客户端 | TLS 反向代理、可信 Host/代理、Worker→Provider 全链路 SSE flush/buffering/timeout 核对、网络策略、认证与安全 headers | | PostgreSQL | 单实例、named volume、迁移/故障测试 | 托管/HA、TLS、最小权限角色、加密、备份/PITR、RPO/RTO 与真实恢复演练 | | Redis | 单实例 AOF、非权威通知层 | 认证/TLS、HA/容量/保留策略、监控；继续保持 DB 事实来源 | -| Worker | 租约/心跳/fencing/重试/取消；数据库权威四层治理、attempt ledger、背压和公平 slice；真实双 Worker Mock 基线 | 滚动排空自动化、更高 Worker 数/真实 Provider 的独立容量与成本规划 | +| Worker | 租约/心跳/fencing/重试/取消；数据库权威四层治理、attempt ledger、背压/公平 slice；DB-time scan/claim/progress/heartbeat aggregate；真实双 Worker Mock 基线 | 滚动排空自动化、更高 Worker 数/真实 Provider 的独立容量与成本规划 | | Secrets | Web write-only；AES-GCM 密文入库；API/Worker 共享独立 keyring；legacy env 可用 | 身份/对象授权、KMS/HSM、短期凭据、批量重加密、轮换审计与每进程最小权限 | | SSRF/数据外发 | 远端 HTTPS、HTTP 仅 loopback、禁重定向与有界正文 | allowlist、DNS/IP 验证、出站代理、元数据阻断、外发审批 | -| 可观测性 | 应用 JSON 日志、组件健康、DB gauges、typed audit、历史 counters/Run 延迟、逐题安全 Provider metadata | 统一运行时日志/traces、告警发送器、生产 SLO、数据库管理员级不可篡改审计 | -| 数据保护 | 显式单向 importer 与 hash 对账 | 保留/删除策略、静态加密、备份/PITR、灾备演练、受控导出 | +| 可观测性 | 受控 JSON 日志源、组件健康、DB gauges、typed audit/history、固定低基数 Prometheus exporter、8 条规则、Worker progress、逐题安全 Provider metadata | 受认证 Dashboard、统一 traces、告警发送/值班集成、生产 SLO、数据库管理员级不可篡改审计 | +| 数据保护 | 13 表单向 importer/hash 对账；canonical audit archive 的 offline verify/精确 restore-delete | 静态加密、签名/WORM、备份/PITR、灾备演练、合规删除与受控导出 | | 供应链 | lockfile、基础 CI、版本标签镜像 | Action SHA/镜像 digest、漏洞门禁、SBOM、签名与 provenance | | 性能/HA | 真实故障正确性验收、指定硬件/commit 的 PostgreSQL16/Redis7/双 Worker Mock 基线，以及 clean `b6a35fe…` 的固定单机 v2 1+5 资格 evidence | 另做真实 Provider 基线、多主机故障、滚动升级和恢复时间验证；现有结果不是生产 SLA/HA | diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index b3fb3f5..0fd428b 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -1,126 +1,95 @@ -# 下一任务：Phase 2 可观测性、保留与恢复闭环 +# 下一任务：完成 P2-06 仓库门禁，再推进 P2-07 恢复闭环 -> 状态：`ready`；治理/审计与 P2-01 已交付，Phase 2 仍保持 `in_progress`，现在可以开始 P2-06/P2-07 +> 状态：`in_progress`；P2-06 功能、主要本地门禁、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已完成，独立 commit/clean-SHA Compose/push 与精确 SHA CI 尚未完成 > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) -> 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0012](decisions/ADR-0012-single-host-slo-capacity-qualification.md)、[ADR-0013](decisions/ADR-0013-stable-image-content-fingerprint.md)、[ADR-0014](decisions/ADR-0014-dual-backlog-slo-profile.md) +> 当前计划：[Phase 2 可观测性与审计保留](plans/2026-08-28-phase-2-observability-retention.md) +> 当前日志：[2026-08-28 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) +> 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) -## 现在从哪里继续 +## 当前事实 -Phase 2 治理/审计垂直切片已经作为实现 SHA `665244e095905083b606b8e98e946ed1a02dc0fc` 提交并 push：Alembic `20260827_0004`、六类治理/审计表与 12 表 importer、global/provider/model/run 四层 per-attempt 治理、固定窗口 RPM/TPM 和 lifetime request/Token/USD budget、policy/hash 与 Run override 冻结、materialized counter/ledger 漂移 fail-closed、confirmed pre-send retry generation、有限 question quantum/backlog、公平排序、typed audit/history、Provider metadata、credential audit、前端治理状态与真实 PostgreSQL 竞争测试。P2-01 随后在独立实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 交付 ADR-0012～0014、v2 四 cell 多轮资格、恢复/连接模型和 exact-project cleanup。 +P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，clean SHA 上的 `P2-local-control-plane-v2` 完成 1 warm-up + 5 measured、23/23 SLO，aggregate SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。这些结果只属于对应历史 SHA，不能替代当前 P2-06 工作树门禁。 -治理 SHA 的增强 capacity evidence SHA-256 为 `40deadebc357bbb24a07c91b05eb39f3d2fb7de11a28da9a7f95871c7acd0588`；完整 acceptance 9/9，evidence SHA-256 为 `ab311665ff0cb834efdd648cd634f943a4cbc5b8b00728ac8597a288a877ddec`。P2-01 冻结树的本地门禁为后端 `829 passed, 29 skipped`、前端 `38 passed`，lint、Mock smoke、前端 build、Alembic 与 Compose config 均通过；`P2-local-control-plane-v2` 在 clean SHA 上完成 1 warm-up + 恰好 5 measured、23/23 SLO 和逐轮 hard invariant/cleanup，aggregate SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`。GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 对同一实现 SHA 4/4 成功；证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 也已 push，[run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。P2-01 已完成，现在直接继续 P2-06/P2-07；不得重复候选或资格工作，也不得跳到 Phase 3。 +当前 P2-06 已按 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 实现： -## 已完成的候选门禁合同 +- Alembic head `20260828_0005` 新增 `worker_processes` 和 bounded audit scan indexes；SQLite→PostgreSQL importer 扩展为 13 表精确 digest，拒绝 live generation 并复制 stopped/stale facts；表非空时 `0005 -> 0004` 在 DDL 前拒绝。 +- 长运行 Worker 注册唯一 generation，只在真实 scan/claim/lease-heartbeat/progress 后按 DB UTC 合并刷新；`/tasks/metrics` 和 exporter 只公开 expected/registered/live/stalled/shortfall 与聚合时间。dependency probe 固定声明不检查 main-loop progress。 +- `GET /api/v1/metrics/prometheus` 输出固定 Prometheus text `0.0.4` gauge，使用一个 DB-time 快照、有界 15 分钟 audit 与 1 小时 latency 窗口、固定 enum label、整次 fail-closed 和每 API 进程 single-flight。 +- `deploy/observability/` 提供固定八条告警规则、抓取示例与对应 Operations Runbook；仓库不部署 Prometheus、Alertmanager、OTel 或通知发送器。 +- `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 提供 canonical JSONL v1、严格文件/权限/大小/schema/hash/rollup 校验、真正离线 verify、精确 digest 绑定、默认不删除和 commit-outcome 分类。它不提供签名/WORM，也不替代整库+keyring backup。 +- 全生产日志调用受字面量/无格式参数静态门禁；第三方动态消息固定化、结构化数值有限化，raw Uvicorn access handler 关闭。 -以下合同已在 `665244e…` 上满足，保留作为后续修改必须重跑的回归基线。 +当前验证事实：合并定向套件全绿，并包含 importer committed-target integrity、retention no-op postverify、外部 logger extra、FIFO/line cap 与 PostgreSQL advisory/row-lock 终审回归；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 后 integration 为 `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、frontend production build、`docker compose config --quiet` 与 `prom/prometheus:v3.5.0` 的八规则 `promtool check rules` 通过。Dirty Compose acceptance 已 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`（SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`），Worker gauges `2/2/2/0/0`，`0005` populated refusal、isolated populated `0004` refusal、两层空库往返和 cleanup C/V/N empty 均成功。最新 dirty capacity 也通过，evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`（SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`）：18 Runs/270 Responses/270 question executions/271 reservations/1229 audit，0 error/drift/duplicate/PEL/lag，Worker expected 2 且 cleanup C/V/N/image 全空；这是 offline Mock 非 SLO 观察值。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。最新 hydration/import integrity 目标集 `67 passed`。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后未擅自迁移。当前仍无 P2-06 独立 commit、clean-SHA Compose、远程 SHA 或 CI 链接；不得把 P2-06 写成 `completed`。 -### 1. 保护工作树并做定向回归 +## 立即执行：P2-06 仓库级收尾 -- 阅读 README、AGENTS、本文件、PROJECT_STATUS、ROADMAP、Phase 2、计划/工作日志及 ADR-0009/0010/0011。 -- 运行 `git status --short --branch`，确认没有与用户工作重叠；不得回滚或格式化无关改动。 -- 先运行治理 repository、Adapter、Runner、API/audit、credential、migration/importer、capacity-script 单测。 -- 特别确认：scope/minute materialized counter 高报与低报均 fail closed；policy/hash 与 Run override 漂移不能绕过限额；`max_retries=0` 的 confirmed pre-send release 仍从未发送 ordinal 恢复。 +### 1. 完成实现终审 -### 2. 真实 PostgreSQL integration +- 复核 Worker registration/stop/late-flush、scan/claim/progress/heartbeat event 定义，确认 timer 无事件时零写入、stale 边界 `last_seen_at == cutoff` 为 live。 +- 复核 exporter cancellation/single-flight：请求取消后 DB 线程终止前不得提前释放 gate；audit 第 `50,001` 行必须整次 `503`，不能截断；数据库/audit/renderer 错误不得返回部分 exposition。 +- 复核 archive 从同一已打开 descriptor 完成 verify/delete/restore，`verify` fresh process 在无效 DB 配置下仍不导入/创建 engine；delete/restore 只处理 archive 列出的精确完整事实，不能做宽 cutoff 删除。 +- 复核 `0005`、prepare fingerprint、13 表 importer、Compose expected Worker count 与 acceptance/capacity 两 Worker 拓扑一致；终审确认 importer committed-target canonical integrity、retention no-op postverify 和 PostgreSQL advisory/row-lock 门禁继续生效。 +- 检查所有生产 logger 调用、第三方 handler、Uvicorn access 和固定错误，确认不反射异常、DSN、路径或 argv marker。 -在隔离 PostgreSQL 运行全部 integration marker，至少实际覆盖并记录： +### 2. 完成剩余验证 -- global/provider/model/run concurrency 竞争； -- 四层 RPM/TPM 原子限制； -- global/run lifetime request、Token、USD budget； -- 并发 backlog 精确 admission 数和 typed `run_backlog_full`； -- finish settlement 与 lease reconciliation 竞争只有一个终态事实； -- audit replay 幂等、同 key 不同 payload fail closed； -- active policy 并发切换、Run 创建/Model 敏感更新锁序； -- `0004` PostgreSQL upgrade/check/downgrade guard/upgrade 和 12 表 importer 对账。 - -不得用 SQLite 单测替代上述真实 PostgreSQL 证据；所有 Provider 行为仍只用 Mock/Stub。 - -### 3. 增强 capacity 与 acceptance - -在同一个冻结候选 commit 上运行增强 `make phase2-capacity`： - -- policy read-back 的所有 concurrency/RPM/TPM/lifetime 数值必须有限； -- Run input reservation、output limit、Token/cost budget 必须有显式上界； -- question quantum 必须小于 15 题 Demo Run，并由 durable audit 证明 cooperative yield； -- 停止 Worker 后并发提交必须得到精确 backlog-limit 个 `202` 和其余 typed `429`，随后恢复 Worker 并 drain 到零； -- 高流量 Model 持续竞争时，低流量 Model 必须在高流量 backlog 排空前取得 claim/slice； -- 比较一/二 Worker 的吞吐与 p50/p95/p99，并验证 Worker loss、Redis interruption、ledger/audit/counter 对账和完整清理； -- evidence 必须记录候选 commit SHA、脚本 SHA-256、环境、配置、原始脱敏计数和限制说明，不得冒充生产 SLA。 - -`phase2_acceptance.py` 的脚本单测为 `19 passed`，完整 `make phase2-acceptance` 已在冻结候选上验证以下三条确定性数据库 seam injection 与真实 Worker/Redis 恢复： - -1. reservation 已提交、`send_started` 尚未提交：接管只能 `released_pre_send`，不得消耗未发送 retry； -2. `send_started` 已提交、settlement 尚未提交：接管必须 conservative settlement，释放并发但不按零退回预算； -3. Provider response 已返回、Response/settlement 本地 commit 尚未确认：本地唯一事实不得 double-count，并明确外部调用/费用可能重复的 at-least-once 边界。 - -这些场景明确是 deterministic database seam injection，不宣称 `SIGKILL` 精确命中亚毫秒边界；一般的“Worker 在若干 Response 后 SIGKILL”也不能替代它们。任一完整 Compose 场景失败或未运行都必须保留为未通过/遗留，Phase 2 继续 `in_progress`。 - -### 4. 全量门禁、commit、push 与精确 SHA CI（已完成） - -至少运行并记录： +以下门禁已经执行并记录，无后续代码变化时不需要为碰绿重复运行： ```bash make lint make test make smoke -make phase2-capacity -make phase2-acceptance -(cd backend && uv run alembic upgrade head) -(cd backend && uv run alembic check) +(cd backend && uv run pytest -m integration) +(cd frontend && npm run build) docker compose config --quiet -git diff --check ``` -- 检查 staged diff、调试残留、生成物、真实密钥/Authorization/Cookie、审计 payload 和文档虚假完成标记。 -- 形成一个独立、可审查的 Phase 2 候选 commit，push 到 `origin/codex/complete-evaluation-workflow`，继续使用当前 PR #2；禁止 force push。 -- 等待该精确 commit SHA 的四个 GitHub Actions 必需 job 全部成功。任何失败都读取日志、修复、创建新 commit、push 并等待新 SHA；不能用本地通过替代远程绿色。 -- commit/SHA、branch、Actions URL、job 结论、capacity/acceptance artifact path 与 SHA-256、命令计数和清理结果已补入 Changelog、Project Status、Phase 2、计划与工作日志。 +迁移验证使用隔离临时 SQLite 和 PostgreSQL；不要为文档收尾擅自 upgrade 默认用户 SQLite。当前剩余命令/检查是： -### 5. P2-01 单机控制面资格（已完成） - -- 历史 v1 clean-SHA 1+5 aggregate 只通过 15/18，继续永久记录为 `failed/not_qualified`；未删除 measured-02，也未只重跑失败 cell。 -- ADR-0014 在任何 v2 样本前冻结 warmed `3/5/8s` 与 cold `6/8/10s` 两个 AND backlog 门禁、两个 distinct validated Worker、22/330/330/331 对账和 project-image cleanup。 -- clean SHA `b6a35fe…` 的全新 v2 invocation 保留 1 warm-up + 5 measured，23/23 SLO 全部通过，容量模型为 `qualified`。该结论只适用于 evidence 记录的 Mock-only 单机 profile，不是生产、真实 Provider、HA、费用或 exactly-once 证明。 +```bash +make phase2-capacity +# dirty acceptance 已通过；commit 后仍须在 clean SHA 重跑两项 +make phase2-acceptance +git diff --check +``` -## 当前工程任务：Phase 2 正式闭环 +- 自动化只能使用 Mock/MockTransport/stub，不调用真实或付费 Provider。 +- `prom/prometheus:v3.5.0` 内的 `promtool check rules` 已验证八条规则成功；最终 diff 只需确认规则文件未再变化，若变化则必须重跑。 +- 已完成的真实 PostgreSQL 路径覆盖 `0005` upgrade/check/空库往返、audit retention advisory/row lock 往返和 13 表 importer；dirty acceptance 已证明 application populated `0005 -> 0004` 拒绝、isolated populated `0004 -> 0003` 拒绝及两层空库往返。clean SHA 仍须重跑并绑定 evidence。 +- 检查 Markdown 内部链接、固定八条 Runbook anchor、exposition metric 名、规则引用、文档版本和实际实现一致。 -治理切片与 P2-01 已交付，证据文档 commit `875f13a…` 的精确 SHA CI 也已 4/4 成功。现在按 P2-06、P2-07 推进；下列范围全部完成后，Phase 2 才能评估是否改为 `completed`。 +### 3. Commit、push 与精确 SHA CI -### P2-06：Exporter、告警、retention 与 Worker progress +- 复核完整 diff/staged diff，排除真实 Key、Authorization/Cookie、credential DSN、ciphertext/nonce/keyring、archive 产物、调试输出和无关生成物。 +- 修复后 staged 技术/安全终审和最新 dirty capacity 已通过；形成一个独立、可审查的 P2-06 commit，在该 clean SHA 上重跑 capacity/acceptance 并绑定 evidence，再普通 push 到 `origin/codex/complete-evaluation-workflow`，继续通过 PR #2 触发 CI；禁止 force push。 +- 等待该**精确 commit SHA** 的四个必需 GitHub Actions job 全部成功。任何失败都读取日志、修复、形成新 commit/push，再等待新 SHA；历史绿色或仅本地通过不能替代。 +- 绿色后再把 P2-06 状态改为 `completed`，并补齐 commit、branch、Actions URL、命令、测试数、跳过/未运行项、Compose cleanup、secret scan 与 `git status --short` 证据。Phase 2 仍因 P2-07 保持 `in_progress`。 -- 为现有 DB gauges、typed counters 和 latency 提供受控 exporter；定义 label cardinality、抓取失败、数据库压力和脱敏边界。 -- 为 backlog、dead-letter、governance integrity、overdraw、queue degraded、Worker stalled 和恢复时长定义告警规则、持续时间、严重度、静默与 Runbook 链接。 -- 增加 Worker 主循环 progress/liveness 事实，例如最后 scan/claim/heartbeat/progress 的 DB-time 证据；不能继续把 dependency probe 冒充 Worker 正在推进。 -- 定义 audit retention class 的实际保留、archive、删除/校验、恢复与失败语义；archive 不得含 Key、ciphertext、nonce、URL、题目/prompt/response正文，且不能宣称不可篡改存储。 -- 若加入 resume canary 独立事件，先用固定安全 enum/bitset 设计 schema；不得写 Provider 控制文本。 +## P2-07：下一独立切片 -### P2-07：备份/恢复与完整运维演练 +P2-06 精确 SHA 全绿后，新建独立计划、工作日志和必要 ADR，完成： -- 演练 PostgreSQL backup→空目标 restore→Alembic head→12 表 count/PK/content fingerprint→managed Run/ledger/audit 可读。 -- 数据库和 keyring 必须独立备份/恢复；既要验证匹配 keyring 可解密，也要验证缺失/错误 keyring fail closed，日志和证据不得回显 Key 或 envelope。 -- 演练 audit archive restore、Redis 重建/consumer group 恢复、Worker 扩缩、dead-letter、commit outcome unknown 和治理完整性告警处置。 -- 补齐尚未覆盖的取消/重试/租约/预算/崩溃组合矩阵，并在真实 PostgreSQL/Redis 下执行。 +- PostgreSQL backup → 空目标 restore → Alembic `20260828_0005` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 +- 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 +- Redis 重建/consumer group 恢复、Worker 扩缩、八条告警响应、dead-letter、commit outcome unknown、governance integrity 与 remaining cancel/retry/lease/budget crash matrix 的真实 PostgreSQL/Redis 演练。 +- audit archive 作为数据库恢复后的精确校验/补回工具参与演练，但不得把 archive 自身 restore 冒充整库、PITR、RPO/RTO 或 WORM 认证。 ## 不变量与非目标 - 不改变 `llmbenchlab-protocol-v1` 的评分、分母、完成率、answered accuracy、排行榜隔离或不可变历史快照。 -- 不调用真实或付费 Provider；自动化只使用 Mock、MockTransport、stub 和故障注入。 -- API/Worker managed Run 受 `0004` 治理；可信本地 CLI 按 ADR-0010 继续 `legacy_unmanaged`，操作者必须独占数据库且没有全局 RPM/TPM/USD 硬保证。 -- Provider 调用不是 exactly-once；本地幂等、ledger 和保守结算不能证明外部调用/账单恰好一次。 -- 不把 Redis、进程内计数或 materialized counter 当成第二事实来源；ledger/DB truth 漂移必须 fail closed。 -- 不把审计称为 WORM；数据库管理员仍可修改数据，读取完整性检查只提供应用层 fail-closed。 -- 不移除 write-only Key、AES-GCM credential、数据库外 keyring、legacy environment、真 SSE、严格 `[DONE]`、nullable `max_tokens` 或既有安全限制。 -- 不新增 Phase 3 Benchmark、代码沙箱、Judge、Arena、Agent、认证、多租户、公共部署或 Kubernetes。 +- API/Worker managed Run 继续以数据库/ledger 为事实来源；Redis、Prometheus、告警、日志、进程内 gate 和 materialized counter 都不是第二状态机。 +- Provider 调用不是 exactly-once；本地幂等、ledger 和保守结算不能证明外部调用或账单恰好一次。 +- 保留 write-only Key、AES-GCM credential、数据库外 keyring、legacy environment、真 SSE、严格 `[DONE]`、nullable `max_tokens` 和既有安全限制。 +- 不新增 Phase 3 Benchmark、代码沙箱、Judge、Arena、Agent、认证、多租户、公共部署、Kubernetes 或生产 HA 声明。 ## Definition of Done -- 已完成切片：治理/审计与 P2-01 已交付；P2-01 已有独立实现 commit、实际 evidence、secret/diff/cleanup 审计和实现 SHA 4/4 CI，证据文档 commit `875f13a…` 自身的精确 SHA CI 也已 4/4 成功；v1 失败与 v2 通过事实同时保留。 -- Phase 2 整体：除上述切片外，Exporter/告警、audit retention archive、Worker progress/liveness、备份恢复和剩余故障矩阵均实现并验证；所有 Phase 2 验收项有可复核证据。 -- 任一关键项未运行或失败时，Phase 2 必须保持 `in_progress`，不得宣称生产 HA、完整可观测性、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +- P2-06：功能、文档、迁移与测试合同一致；全量本地/真实基础设施门禁通过；独立 commit 已 push；该精确 SHA 四个必需 CI job 全绿；状态/计划/日志记录完整且无秘密或生成归档。 +- Phase 2：只有 P2-06 上述门禁和 P2-07 backup/restore、Redis 重建、告警处置与完整故障矩阵均有可复核证据后，才可评估 `completed`。 +- 任一关键项未运行、失败或未形成精确 SHA 远程绿色时，相关任务和 Phase 2 必须保持 `in_progress`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展、WORM 或 Provider exactly-once。 ## 可直接复制给 Codex 的任务指令 ```text -请执行 docs/NEXT_TASK.md 的“Phase 2 可观测性、保留与恢复闭环”。治理/审计候选 `665244e095905083b606b8e98e946ed1a02dc0fc`、P2-01 v2 实现 `b6a35fef1dd069ebb54b69955058915c722aa34d` 与证据文档 `875f13a253c40b7573d45c6287385e60f2bb8f04` 的实际 evidence 和精确 SHA CI 已完成，不要重复实现、重跑碰绿或改写 v1/v2 证据。建立新的工作日志与执行计划，交付受控 exporter/告警、低基数标签、Worker DB-time progress/liveness、audit retention archive/restore；再演练 PostgreSQL 与数据库外 keyring 配对 backup/restore、Redis 重建和剩余故障矩阵。自动化只能用 Mock/Stub，不得调用真实 Provider；保持 protocol-v1、ledger/DB truth、write-only Key 和 fail-closed 边界。每个独立切片都须测试、提交、push 并等待精确 SHA CI；正式闭环未全部满足前 Phase 2 保持 in_progress，不得跳到 Phase 3 或宣称生产 HA/Provider exactly-once。 +继续执行 docs/NEXT_TASK.md。当前 P2-06 功能以及 lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 和修复后 staged 技术/安全终审已通过，不要重做 P2-01，也不要跳到 P2-07。现在形成独立 commit，在 clean SHA 重跑 capacity/acceptance，普通 push 当前工作分支并等待该精确 SHA 的四个 CI job 全绿；失败就修复并对新 SHA 重跑。P2-06 绿色后再独立推进 PostgreSQL+keyring backup/restore、Redis 重建、告警响应和剩余恢复矩阵。自动化只用 Mock/Stub；Phase 2 在 P2-07 完成前保持 in_progress。 ``` diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 503bdc3..ae024a4 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -10,7 +10,7 @@ - 容量方法和当前 Mock 基线：[PERFORMANCE.md](PERFORMANCE.md) - 部署、迁移和凭据：[DEPLOYMENT.md](DEPLOYMENT.md) -- 数据库治理语义：[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[交付边界 ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) 和 [pre-send retry generation ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md) +- 数据库治理语义：[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[交付边界 ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[pre-send retry generation ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md) 和 [可观测性/保留 ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) - API 字段与错误码：[API.md](API.md) - 安全边界：[SECURITY.md](SECURITY.md) @@ -34,6 +34,7 @@ curl -sS http://127.0.0.1:8000/api/v1/ready curl -sS http://127.0.0.1:8000/api/v1/governance/policy curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics curl -sS 'http://127.0.0.1:8000/api/v1/tasks/history?window_hours=24' +curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus ``` ### 2.2 当前 gauges @@ -52,6 +53,8 @@ curl -sS 'http://127.0.0.1:8000/api/v1/tasks/history?window_hours=24' | `dead_lettered` | 失败预算耗尽的 Run | 任一新增都应检查 Run audit 和已有 Response | | `runs_with_queue_notification_error` | 曾发生 Redis publish 失败的 Run | 可由 DB reconciliation 完成，但应检查 Redis 和延迟 | | `total_dispatches` / `total_attempts` | cooperative slice / lease acquisition 累计 | 用历史增量判断调度活动，不是错误计数 | +| `worker_live_processes` / `worker_shortfall_processes` | DB UTC 下仍有真实进展的 Worker / 相对显式 expected minimum 的缺口 | shortfall 持续非零说明主循环停滞、进程退出或部署 expected 配置错误 | +| `worker_last_scan_at` / `worker_last_claim_at` / `worker_last_progress_at` / `worker_last_lease_heartbeat_at` | 未停止 generation 的最近聚合事实 | 空值表示尚无该类事实；时间陈旧不能由 dependency probe 的成功覆盖 | ### 2.3 历史 counters、延迟和单 Run audit @@ -59,14 +62,86 @@ curl -sS 'http://127.0.0.1:8000/api/v1/tasks/history?window_hours=24' `GET /api/v1/runs/{run_id}/audit?offset=0&limit=100` 按 `(occurred_at,id)` 稳定分页。审计是应用 append-only，不是密码学不可篡改或 WORM；数据库管理员仍能修改它。operational 事件至少保留 90 天，security/credential 事件至少 365 天。清理不在请求链路自动执行：必须先归档、核验并生成所需 rollup，再由明确维护操作清理。 -建议外部监控至少实现以下症状规则，具体数值要根据目标环境基线设定： +`GET /api/v1/metrics/prometheus` 提供固定低基数 Prometheus text exposition。它不接受查询参数；每个 API 进程只允许一笔 collection in flight，重叠抓取返回 `429`。数据库、Run/governance、15 分钟 typed-audit window、1 小时 Run latency 与 Worker aggregate 使用同一数据库快照和 DB UTC；Redis 仅为快照外的非权威 availability observation。任一 retained audit 损坏使整次抓取 fail closed，不返回部分指标。所有 series 都是 gauge；rolling-window count 不是 counter，不得对它使用 `rate()` 或 `increase()`。 + +仓库提供 `deploy/observability/prometheus-scrape.example.yml` 和 `deploy/observability/prometheus-alerts.json`，但不部署 Prometheus、Alertmanager 或通知发送器。推荐 scrape interval 为 60 秒且不得低于 30 秒。告警规则固定为下节八条；不要把一次 `/ready=200`、一次 exporter scrape 成功或任意 peer 的 progress 当作当前 Worker 主循环仍在推进的证明。 + +额外的运维观察仍应包括： - 立即告警：数据库不可用、schema 非 head、`overdrawn_governance_scopes>0`、`governance_integrity_error` 新增、审计/ledger 唯一性或 reserved 对账漂移。 - 高优先级：dead-letter 新增、`expired_running` 持续、settlement unknown/保守结算异常增加、真实 Provider 账单超出本地 consumed 上界预期。 - 持续性告警：`managed_backlog`/`due_pending` 多个观察窗口单调增长，`governance_delayed` 跨越应有 fixed-minute 窗口，queue lag/PEL 不下降，p95/p99 明显偏离同硬件基线。 - 容量预警：数据库连接池等待、deadlock/conflict/temp file、Redis 内存/Stream 长度或 Worker CPU/内存逼近环境阈值。 -仓库不包含告警发送器或 Prometheus exporter；必须由部署环境轮询 API/数据库并负责通知。不要把“一次 `/ready=200`”当作 Worker 主循环仍在推进的证明。 +告警发送、路由、值班升级和长期存储仍由部署环境负责。 + +### 2.4 固定告警响应 + +下列 Runbook 与 `deploy/observability/prometheus-alerts.json` 一一对应。响应时先保存告警名称、UTC 起止时间、当前 revision、固定指标快照和相关 Run audit；不得把 archive/raw payload、凭据、DSN、内部 URL 或异常原文贴入工单。静默必须使用窄 job/instance matcher，并记录 owner、ticket 与明确到期时间；禁止全局 silence。 + + +#### Exporter unavailable + +- **响应**：确认 API 进程、`/ready` 的 database/schema 分量和 `/api/v1/metrics/prometheus` 状态；`429` 表示已有 collection，`500` 表示 audit integrity，`503` 表示数据库或 observation cap，不要只重启 Prometheus。 +- **证据**：保存固定错误码、DB/head 状态、API 资源与抓取持续时间；不得保存响应中的未知正文或服务配置。 +- **恢复**：先恢复 PostgreSQL/head 或处置损坏 retained event；确认一次完整抓取成功且没有并发 collection 后再关闭事故。 +- **静默**：仅受控维护可按 job/instance 静默，最长 4 小时；不得掩盖 audit integrity 或数据库不可用。 + + +#### Backlog persistent + +- **响应**：按第 4.1 节检查 Worker DB-time progress、due/deferred、lease、Provider 和连接池，不通过删除队列或手改 Run 解堵。 +- **证据**：保存 backlog/due/running/shortfall、最近 scan/claim/progress、policy hash 与固定窗口 history。 +- **恢复**：停止扩大 admission，恢复 Worker/数据库/Provider 后确认 due backlog 连续下降并最终收敛。 +- **静默**：只允许已批准容量维护的窄 job matcher，最长 4 小时。 + + +#### Dead letter + +- **响应**：立即执行第 4.3 节，读取 Run、Response 和完整分页 audit；不重置失败计数或重新使用旧 operation key。 +- **证据**：保存稳定 reason、failed/max attempts、policy version/hash、已有 Response 数和新建替代 Run 的关联。 +- **恢复**：修复根因后创建新 Run，并确认没有新增 dead-letter；旧事实必须保留。 +- **静默**：捕获证据后才可按 job 短暂静默，最长 1 小时。 + + +#### Governance integrity + +- **响应**：停止新 admission 和相关 Worker 外发，保存数据库快照；从 never-delete ledger 重算 scope/minute projection，禁止直接 UPDATE counter。 +- **证据**：保存固定 integrity reason、revision、policy hash、漂移字段类别与 reservation 状态计数，不复制 payload 或凭据。 +- **恢复**：只能通过经评审的事实修复/恢复路径并重跑完整性测试；确认新 reserve 在重算后成功且旧漂移仍可审计。 +- **静默**：不作常规静默；只有事故负责人明确控制影响时才使用最窄 matcher 和最短期限。 + + +#### Governance overdraw + +- **响应**：停止扩大真实 Provider 流量，核对 actual usage、reservation 上界、价格和 Provider 账单；不要把 actual 改回 reservation。 +- **证据**：保存 scope 类型、非秘密 consumed/reserved 汇总、policy/Run override hash 和外部账单关联编号。 +- **恢复**：修正未来 Run 的显式 Token/cost bound 或 policy；已 overdrawn 事实保留并由治理规则决定是否允许后续流量。 +- **静默**：不作常规静默；告警消失也不能替代账单对账。 + + +#### Queue degraded + +- **响应**：先确认 PostgreSQL 正常，再按第 7 节检查 Redis、Stream、consumer group、lag/PEL 和 Worker DB scan。 +- **证据**：保存 queue configured/available、DB pending/due 与固定的 queue notification error 计数；不保存 Redis URL/密码。 +- **恢复**：恢复 Redis 后验证新通知、DB reconciliation 与 PEL/lag 收敛；不得通过删除 Stream/consumer group 伪造恢复。 +- **静默**：受控 Redis 维护可按 job 静默，最长 4 小时。 + + +#### Worker stalled + +- **响应**：核对部署的 expected minimum、未停止 generation 的 DB UTC last_seen/scan/claim/progress/heartbeat 与进程状态；dependency-only probe 不能否定此告警。 +- **证据**：保存 expected/registered/live/stalled/shortfall 和五类聚合时间，不导出 worker/generation ID。 +- **恢复**：优先 graceful restart/scale，等待旧 lease 自然过期并核对 fencing、Response 唯一性和 reservation reconciliation。 +- **静默**：仅明确缩扩容维护可按 job/instance 静默，最长 1 小时；维护时 expected minimum 必须与目标规模一致。 + + +#### Lease recovery slow + +- **响应**：按第 6.2 节检查 DB 时钟、reaper scan、Worker shortfall、retry backoff 和旧 token ledger，不手工抢占或释放 `send_started`。 +- **证据**：保存 expired-running 数、oldest age、恢复阈值、最近 scan/claim/progress 和固定的 lease/audit 状态。 +- **恢复**：确认 token 递增、旧写入被 fence、reservation 恰好收敛且 Run 进入可解释终态。 +- **静默**：只有已建立恢复 owner 后才可窄静默，最长 15 分钟。 ## 3. 限流和预算 @@ -181,16 +256,18 @@ Run 终态、defer 或 exhaust 转换会先在短事务中提交，再做 lease Compose 扩到两个 Worker： ```bash -docker compose up -d --no-deps --scale worker=2 worker +LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES=2 \ + docker compose up -d --no-deps --scale worker=2 worker ``` 缩到一个 Worker： ```bash -docker compose up -d --no-deps --scale worker=1 worker +LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES=1 \ + docker compose up -d --no-deps --scale worker=1 worker ``` -缩容依赖 SIGTERM grace。先观察活动 Run，等待被停止 Worker 排空；若 grace 耗尽，未完成 lease 留到数据库自然过期，由 peer 以递增 token 接管。缩容后检查 Worker health、`running/expired_running/retry_scheduled`、active reservations 和 Response 唯一性。 +`LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES` 是部署声明，不从历史进程行猜测；规模与 expected 不一致会产生有意的 shortfall。缩容依赖 SIGTERM grace。先观察活动 Run，等待被停止 Worker 排空；若 grace 耗尽，未完成 lease 留到数据库自然过期，由 peer 以递增 token 接管。缩容后检查 Worker health、DB-time registered/live/stalled、`running/expired_running/retry_scheduled`、active reservations 和 Response 唯一性。 ### 6.2 Worker crash/lease expiry @@ -254,7 +331,52 @@ PostgreSQL 不可用时没有可替代事实源。API 不应接受新 Run，Work 若恢复点早于某些 Provider 外发，数据库无法知道备份之后的远端消费；必须以 Provider 账单和 request ID 独立核对。Redis 可以重建通知，不能用来补造丢失的数据库事实。 -## 9. 0004 安全回滚 +## 9. Audit archive、核验与保留 + +`audit_events.expires_at` 表示行已达到最低保留期，不表示可由请求链路自动删除。Operational/security 事件仍分别至少保留 90/365 天；维护流程必须严格执行 **archive → offline verify → reconcile → maintenance-window delete**。Archive 是含内部 ID 的敏感运维文件，hash 只用于误改检测和精确文件确认，不是签名、认证、WORM 或不可篡改证明。 + +先创建由当前维护用户拥有、不可被 group/other 写的目录；建议 `0700`： + +```bash +install -d -m 0700 /approved/internal/audit-archives +llmbenchlab-audit-retention archive \ + --output /approved/internal/audit-archives/audit-2026-08-28.jsonl +llmbenchlab-audit-retention verify \ + --archive /approved/internal/audit-archives/audit-2026-08-28.jsonl +``` + +工具拒绝 symlink、非当前用户所有或权限宽于 `0600` 的输入；输出使用同目录 `0600` 临时文件、no-follow/no-replace 安装和 fsync。单批最多 10,000 条、文件最多 128 MiB；`has_more_eligible=true` 时以新的目标文件重复 archive，不能覆盖既有文件。CLI 成功只输出固定 status/count/SHA-256，不输出路径、event ID 或 payload；不要把 archive、raw child output 或命令中的本地路径复制到共享工单。 + +`verify` 是完全离线操作，不读取数据库配置、不创建 engine，也不访问 Redis。它校验 canonical JSONL、完整 event 存储事实、source revision、排序、rollup、content/file digest 和 payload contract。完成离线核验后，在 API/Worker/audit writers 已停止的维护窗口，用 verify 输出的精确文件 SHA 执行： + +```bash +llmbenchlab-audit-retention reconcile \ + --archive /approved/internal/audit-archives/audit-2026-08-28.jsonl \ + --confirm-sha256 +llmbenchlab-audit-retention delete \ + --archive /approved/internal/audit-archives/audit-2026-08-28.jsonl \ + --confirm-sha256 +``` + +`delete` 只逐条删除已经核验且与数据库完整事实精确相等的 ID/event-key 集合，绝不执行宽泛 cutoff delete；任一缺失、额外或字段漂移会整体拒绝。`restore` 使用同一精确比较和确认 digest，只恢复缺失且无冲突的行： + +```bash +llmbenchlab-audit-retention restore \ + --archive /approved/internal/audit-archives/audit-2026-08-28.jsonl \ + --confirm-sha256 +``` + +退出码 `0` 表示已确认成功，`2` 表示在提交前安全失败，`3` 表示提交已成功但后验核验失败，`4` 表示 commit outcome unknown。遇到 `3` 或 `4` 不得盲目重跑 mutation：保留 archive/count/digest，先执行只读 `reconcile` 判定数据库与 archive 的精确关系，再由操作者决定恢复或删除。Archive 与数据库/keyring 备份是不同资产；archive 不含 credential ciphertext/nonce/keyring，也不能单独恢复 stored Provider Key。 + +## 10. 0005 与 0004 安全回滚 + +### 10.1 Worker progress / retention revision 0005 + +revision `20260828_0005` 增加 `worker_processes` 和 audit retention/exporter 扫描索引。`0005 -> 0004` 在第一条 DDL 前检查 Worker process facts；只要表中存在任何 generation 行就拒绝，以免静默丢失注册、进展或 graceful-stop 证据。安全降级必须先停止 Worker、归档需要保留的 process facts，并由明确的数据生命周期审批清空；正常代码回滚应保留 0005 schema 并优先向前修复。 + +正式迁移验收同时保留两层独立门禁：populated 0005 数据库拒绝跨过 Worker progress revision且 revision/核心事实不变；隔离空库允许 `0005 -> 0004 -> 0005`。这不会替代下节已有的 populated 0004 governance 拒绝与空库 `0004 -> 0003 -> 0004` 往返。 + +### 10.2 Governance/audit revision 0004 revision `20260827_0004` 增加 policy、scope、minute bucket、question execution、Provider attempt ledger、typed audit、Run governance/fairness 字段及 Response Provider metadata。downgrade guard 在第一条 DDL 前检查数据损失风险。 @@ -278,7 +400,7 @@ revision `20260827_0004` 增加 policy、scope、minute bucket、question execut 隔离空数据库允许 `0004 -> 0003 -> 0004` 往返；真实验收同时证明 populated 数据库 downgrade 被拒绝且 revision/core protocol hash 不变。详见 [PERFORMANCE.md](PERFORMANCE.md) 和 Phase 2 acceptance evidence。schema downgrade 不是 PostgreSQL→SQLite 回迁。 -## 10. 事故收尾清单 +## 11. 事故收尾清单 - [ ] Run/Response 状态只由 PostgreSQL事实解释，没有手工覆盖。 - [ ] Redis PEL/lag 已观察，未通过删除权威数据“修复”。 @@ -288,5 +410,8 @@ revision `20260827_0004` 增加 policy、scope、minute bucket、question execut - [ ] dead-letter、conservative settlement、failed attempt 和 Provider 账单已关联。 - [ ] policy version/hash、Run frozen snapshot 和协议/data hash 已记录。 - [ ] 日志、工单和 evidence 不含 Key、DSN、keyring、密文或原始 Provider body。 +- [ ] Worker expected minimum 与实际部署规模一致；registered/live/stalled/last-progress 使用同一 DB UTC 解释，dependency probe 未被当作主循环进展。 +- [ ] Prometheus rolling-window gauge 未被当作 counter；告警 Runbook、owner/ticket/expiry 和窄 silence 均有记录。 +- [ ] 过期 audit 已先 archive、离线 verify 并按精确 digest/reconcile 操作；没有 cutoff 宽泛删除，commit unknown 未被盲目重试。 - [ ] 修复后重跑目标测试、真实 Compose acceptance 和同硬件容量基线。 - [ ] 清理只作用于明确的隔离测试 project；生产 evidence/backup 按保留策略归档。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index aaf43c4..e49c0bc 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -6,7 +6,7 @@ - Phase 0 — 项目治理和架构：`completed`（2026-08-24） - Phase 1 — MVP 垂直链路：`completed`（2026-08-25） -- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计与 P2-01 单机资格已完整交付；P2-06/P2-07 运维闭环仍有缺口） +- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计与 P2-01 单机资格已完整交付；P2-06 已实现但仓库级收尾仍进行中，P2-07 运维恢复仍有缺口） - Phase 3 — 标准 Benchmark 与代码评测：`in_progress`（仅可信本地 MMLU-Pro/GPQA-Diamond 客观题提前切片） - Phase 4–6：`planned` @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`，PR [#2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)。P2-01 v2 实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 已普通 push，GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 的四个必需 job 均成功；证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 也已普通 push，其精确 SHA 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 4/4 成功。后者完成仓库级收尾，不改变性能证据绑定的实现 SHA。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`，PR [#2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)。P2-01 v2 实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 已普通 push，GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 的四个必需 job 均成功；证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 也已普通 push，其精确 SHA 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 4/4 成功。后者完成 P2-01 仓库级收尾，不改变性能证据绑定的实现 SHA。P2-06 当前改动尚未形成独立 commit/push，也没有可引用的精确 SHA CI；不得把上述历史远程门禁归因于当前工作树。 ## 已交付基线 @@ -41,10 +41,19 @@ - 精确 SHA `665244e…` 的增强 capacity 使用有限 policy、PostgreSQL 16、Redis 7 与两个 Worker 完成：并发 backlog 精确为 4 个 `202` + 2 个 typed `429`，cooperative yield 与跨 Model 公平顺序均有 durable audit 证据，最终 18 Runs/270 Responses/271 ledger/1229 audit 对账且无 active/reserved/overdrawn 漂移。 - P2-01 的 `P2-local-control-plane-v2` 已在干净 SHA `b6a35fe…` 从零执行 1 次 warm-up + 恰好 5 次 measured trial，本次 invocation 的 `discarded_trials=0`；四个 cell、23/23 SLO 与每轮 hard invariant 全部通过，容量模型为 `qualified`。aggregate SHA-256 为 `a76d167b…d0d9`，六轮均精确完成 22 Runs/330 Responses/330 QuestionExecutions/331 reservations，并清理本项目容器、卷、网络和唯一 build image。历史 v1 aggregate `f993c11f…e3b2` 继续保持 `failed/not_qualified`。 +## 当前 P2-06 实现（仓库级收尾进行中） + +- [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；Alembic head 已在当前工作树扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 +- `GET /api/v1/metrics/prometheus` 已实现固定 Prometheus text `0.0.4` gauge：一个 DB-time 读快照、15 分钟 typed-audit 窗口、1 小时 Run latency、硬读取上限、固定 enum label、整次 fail-closed 与每 API 进程 single-flight。`deploy/observability/` 提供固定八条规则和安全抓取示例；仓库不部署 Prometheus、Alertmanager 或通知发送器。 +- `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 已实现 canonical JSONL v1、严格权限/大小/行/schema/hash/rollup 校验、离线 verify、精确 digest 绑定、默认不删除、双方言事务与 commit outcome 分类。Archive 是敏感运维文件，hash 只用于完整性/绑定，不是签名或 WORM，也不替代 P2-07 的数据库+keyring 备份。 +- importer 当前合同为 `0005` 的 13 表精确 count/PK/content digest；live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 +- 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 +- 上述实现的合并定向套件已全绿；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck），`make test` 为后端 `916 passed, 33 skipped`、前端 `38 passed`，`make smoke` 为 `1 passed, 7 deselected`，临时 PostgreSQL 16/Redis 7 migration/check 和真实 integration 为 `33 passed, 0 skipped`，临时 SQLite head→0001→head/check、frontend production build、`docker compose config --quiet` 与临时 `prom/prometheus:v3.5.0` 的八规则 `promtool check rules` 均通过。Dirty 工作树 Compose acceptance 已 9/9 通过，Worker gauges 为 `2/2/2/0/0`，两级 populated downgrade refusal、两层空库往返和 cleanup 空检查均成功；最新 dirty capacity 也通过，最终 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit，0 error/drift/duplicate/PEL/lag 且 cleanup 全空。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。最新 hydration/import integrity 目标集 `67 passed`。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。独立 commit、clean-SHA Compose、普通 push 与精确 SHA 四个必需 CI job 仍待完成。 + ## 仍未完成 -- P2-06：受控 metrics exporter、告警规则/响应、audit retention archive/restore、Worker DB-time progress/liveness、全日志源治理；现有 dependency probe 不能证明主循环正在推进。 -- P2-07：PostgreSQL backup/restore、数据库与 keyring 配对恢复、audit archive/Redis 重建、剩余故障矩阵和完整运维演练。 +- P2-06：实现与 lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 及修复后 staged 技术/安全终审已通过，但独立 commit、clean-SHA Compose、push 与精确 SHA 远程 CI 尚未完成；在这些门禁绿色前保持 `in_progress`。 +- P2-07：PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练。P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 @@ -55,7 +64,7 @@ - trusted-local CLI 按 [ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) 继续 `legacy_unmanaged`，没有全局 RPM/TPM/USD 硬保证；操作者必须停止常规 API/Worker 并独占数据库。 - audit 是应用 append-only、event-key 幂等且 read 时校验 schema/hash，但数据库管理员仍可修改，不能宣称 WORM。 - Provider metadata 不安全时归一化为 `null`；credential audit 不保存 origin。Key、Authorization、ciphertext、nonce、keyring、Provider URL、题目/prompt/response正文均不得进入 audit。 -- Worker probe 只检查数据库/head/Redis 能力，不证明主循环仍在 scan/claim/heartbeat/progress；这是正式 Phase 2 closure 缺口。 +- Worker probe 只检查数据库/head/Redis 能力，不证明主循环仍在推进；Worker 主循环事实现在由 DB-time progress 聚合公开。没有 exact generation handoff 前，probe/容器 healthcheck 仍不得冒充当前进程 event-loop liveness。 - importer 会复制完整敏感评测内容和 credential ciphertext；只支持停写源→空目标单向导入。keyring 不随数据库复制，exit 3/4 禁止盲目重试。 - 远程 Provider 只允许 HTTPS（HTTP 仅 loopback），但仍无 destination allowlist、DNS rebinding 防护、出站隔离、认证、TLS 终止、生产 KMS 或多租户安全；不得直接暴露公网。 - 当前 Python 3.14 本地测试仍可能显示上游弃用 warning；CI 固定 Python 3.12。Vite build 仍有既有 Recharts 大 chunk warning。 @@ -64,6 +73,19 @@ | 验证 | 实际结果 | 当前结论 | | --- | --- | --- | +| P2-06 合并定向套件 | 全绿；不在最终全量前记录或推断全量测试数 | 目标实现回归通过，不替代完整 `make test` | +| P2-06 `make lint` | Ruff 152 files、format check、ESLint、TypeScript typecheck 全绿 | 当前未提交工作树通过 | +| P2-06 `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed` | 当前未提交工作树通过；只用 Mock/Stub | +| P2-06 Mock smoke | `1 passed, 7 deselected` | 完全离线通过 | +| P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；尚未绑定远程 SHA | +| P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；默认用户 SQLite 非 head 的 check 失败后未迁移 | current head `20260828_0005`；用户数据保持原状 | +| P2-06 build/config | frontend build 成功（保留 662.39 kB chunk warning）；Compose config exit 0 | 从根目录误跑 npm 的失败已记录并用正确目录重跑通过 | +| P2-06 Prometheus 规则 | `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | 临时容器验证；仓库仍不部署 Prometheus/Alertmanager | +| P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`、`0005`/isolated `0004` populated refusal、两层空库往返、cleanup C/V/N empty | +| P2-06 dirty capacity | 最新 artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453` 通过 | `dirty=true`；18/270/270/271/1229，0 error/drift/duplicate/PEL/lag，Worker expected=2、cleanup C/V/N/image=0；offline Mock、非 SLO | +| P2-06 补充静态检查 | 过宽 `scripts/` Ruff 命令暴露 93 条既有 modernization 告警；`--select E,F,I` 通过 | 如实保留首次结果，不把范围外历史告警归为本次回归 | +| P2-06 staged 技术/安全终审 | structured-extra High 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 当前 dirty 工作树终审通过 | +| P2-06 剩余门禁 | commit、clean-SHA Compose、push、精确 SHA CI 待完成 | 不得标记仓库级收尾完成 | | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 仍 open，未合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | @@ -77,7 +99,7 @@ | 正式 v2 单机资格 | 精确 `b6a35fe…`，1+5、23/23，aggregate SHA-256 `a76d167b…d0d9` | passed/qualified；Mock-only 单机控制面，不是生产或真实 Provider SLA | | 真实 Provider | 未运行（有意） | 所有自动化只使用 Mock/Stub/MockTransport | -详细命令与限制见 [当前工作日志](worklogs/2026-08-28-phase-2-slo-capacity-model.md) 和 [TESTING.md](TESTING.md)。 +详细命令与限制见 [当前 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) 和 [TESTING.md](TESTING.md)。 ## 最近工作日志 @@ -88,7 +110,8 @@ - [OpenAI-compatible SSE](worklogs/2026-08-27-openai-compatible-sse-streaming.md) - [Phase 2 治理、审计与性能](worklogs/2026-08-27-phase-2-governance-audit-performance.md) - [Phase 2 正式 SLO 与容量模型](worklogs/2026-08-28-phase-2-slo-capacity-model.md) +- [Phase 2 可观测性与审计保留](worklogs/2026-08-28-phase-2-observability-retention.md) ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 是当前合同：治理/审计候选已交付，P2-01 v2 也已完成独立实现提交、正式本机证据、实现 SHA CI 和证据文档 SHA CI；下一工程范围是 P2-06 的 Exporter/告警/retention/Worker progress，以及 P2-07 的 backup/restore 和剩余故障演练；Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 是当前合同：先完成当前 P2-06 工作树的独立 commit、clean-SHA Compose、push、精确 SHA CI 与证据收尾，再单独推进 P2-07 的数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 768d9b5..68309a0 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,7 +30,7 @@ | --- | --- | --- | --- | --- | | Phase 0 | 项目治理和架构 | 可执行的需求、架构、协议、ADR 与持续文档流程 | `completed` | [PHASE-0-GOVERNANCE.md](phases/PHASE-0-GOVERNANCE.md) | | Phase 1 | MVP 垂直链路 | Mock 模型到 Run、逐题结果与排行榜的离线闭环 | `completed` | [PHASE-1-MVP.md](phases/PHASE-1-MVP.md) | -| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计与 P2-01 单机资格已交付；P2-06/P2-07 运维闭环待完成 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | +| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计与 P2-01 已交付；P2-06 已实现待仓库级收尾，P2-07 恢复闭环待完成 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | | Phase 3 | 标准 Benchmark 与代码评测 | 已有 MMLU-Pro/GPQA 可信本地切片；IFEval、沙箱与完整插件体系待完成 | `in_progress` | [PHASE-3-BENCHMARKS.md](phases/PHASE-3-BENCHMARKS.md) | | Phase 4 | Judge、Arena 与长上下文 | 可校准 Judge、Pairwise Judge、个人 Arena 和长上下文评测 | `planned` | [PHASE-4-JUDGE-ARENA.md](phases/PHASE-4-JUDGE-ARENA.md) | | Phase 5 | Agent、私有与 Live Benchmark | 工具调用轨迹、隔离私有集和持续更新的 Live Benchmark | `planned` | [PHASE-5-AGENT-LIVE.md](phases/PHASE-5-AGENT-LIVE.md) | @@ -179,11 +179,11 @@ - PostgreSQL 作为共享部署数据库和任务唯一事实来源；SQLite 保留单 Worker 本地兼容，并提供显式、单向、可对账的 SQLite→PostgreSQL 导入工具。 - Redis Streams 只提供 at-least-once 低延迟通知；任务状态、取消、租约、重试和 dead-letter 均由数据库裁决，Redis 故障由数据库扫描恢复。 - 独立 Worker、原子领取、数据库时间租约、心跳、单调 fencing token、逐题幂等、有限重试、取消、过期接管和 dead-letter；大 Run 快照加载移出事件循环并保持租约心跳，dead-letter 前从持久化 Response 重聚合证据。 -- Alembic `0004` 的 policy/scope/minute bucket/question execution/Provider reservation/audit event 六类表、Run/Response 证据字段与 12 表 SQLite→PostgreSQL importer。 +- Alembic `0004` 的 policy/scope/minute bucket/question execution/Provider reservation/audit event 六类治理表及 `0005` 的 Worker process/progress 表与 bounded audit indexes；Run/Response 证据字段和 13 表 SQLite→PostgreSQL importer。 - managed Run 冻结 policy/hash 与显式 overrides；global/provider/model/run 四层 concurrency、固定窗口 RPM/TPM、global/run lifetime request/Token/USD budget 和逐 Provider HTTP attempt ledger。 - materialized counter 只作 ledger 投影；counter、policy/hash 或 Run override 漂移 fail closed。confirmed pre-send release 按 ADR-0011 不消耗未发送 HTTP retry。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model 公平排序。 -- typed audit、Run audit、task history/latency、Provider metadata、credential 非秘密事件和前端治理状态；Exporter/告警、retention archive 与 Worker progress/liveness 仍待完成。 +- typed audit、Run audit、task history/latency、Provider metadata、credential 非秘密事件和前端治理状态；当前工作树另已实现固定低基数 Prometheus exporter/八条规则、canonical retention archive/verify/reconcile/restore/delete、Worker DB-time progress/liveness 聚合与全日志源治理。 - enhanced Mock-only capacity/acceptance、真实 PostgreSQL 竞争测试，以及绑定 clean SHA 的正式多轮单机控制面资格；backup/restore 仍待完成。 ### 非目标 @@ -202,11 +202,11 @@ | ID | 状态 | 已交付 / 剩余范围 | | --- | --- | --- | | P2-01 一致性与容量设计 | 已交付 | ADR-0012～0014、DB truth/lease/fencing/治理、v2 四 cell 多轮统计、恢复与连接模型已实现；clean SHA `b6a35fe…` 的 1+5 资格为 23/23、`qualified`；证据文档 `875f13a…` 的精确 SHA CI 4/4 成功 | -| P2-02 PostgreSQL 迁移 | 切片已实现 | `0002`/`0003` 可靠性与凭据基础、`0004` 六类治理/审计表和 12 表 importer 已实现；本地及精确 SHA 远程 PG migration/check/integration 已通过，无自动反向回迁 | -| P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付 | +| P2-02 PostgreSQL 迁移 | 切片已实现 | `0002`/`0003` 可靠性与凭据基础、`0004` 六类治理/审计表已通过既有门禁；当前 P2-06 工作树增加 `0005`、13 表 importer、live-Worker preflight 与 downgrade guard，尚待本切片全门禁/精确 SHA CI | +| P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付；当前 P2-06 工作树增加 generation 级 DB-time progress，dependency probe 仍只检查 capability | | P2-04 生命周期可靠性 | 可靠基础已交付 | retry/取消/恢复/dead-letter/终态重算及三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；外部调用仍不保证 exactly-once | | P2-05 并发治理 | 切片已实现 | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序和完整性 fail-closed 已实现；真实 PG/capacity/acceptance/精确 SHA CI 候选门禁已通过 | -| P2-06 可观测性 | 切片已实现 | typed audit/history、Run latency、Provider metadata、credential audit 和 UI 状态已实现；Exporter/告警、retention archive、Worker progress/liveness 与全日志源治理未完成 | +| P2-06 可观测性 | `in_progress` | typed audit/history、Run latency、Provider/credential evidence 基线已交付；exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现，lint/test/integration/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已通过；commit/clean-SHA Compose/push 和精确 SHA CI 未完成 | | P2-07 验证与运维 | 切片已实现 | enhanced capacity/PG tests、正式 v2 单机资格、精确候选 evidence 和运维/性能文档已交付；backup/restore、告警响应与完整失败矩阵未完成 | ### 验收标准 @@ -214,8 +214,8 @@ - `delivered`：API 执行中重启和实际租约 owner Worker `SIGKILL` 后，未完成 Mock Run 可恢复，已有 Response ID 保持唯一。 - `delivered`：真实 PostgreSQL 并发领取只有一个有效 lease；自然过期后由递增 fencing token 接管，陈旧 owner 写入被拒绝。 - `delivered/partial`：pending/running 取消有真实 Compose 证据；有限重试、超时和 dead-letter 有自动化状态机/Runner 证据，但尚未把所有失败组合都纳入完整生产式故障演练。 -- `delivered`：12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 -- `implemented/partial`：typed audit/history、Run DB-time latency、Provider metadata、credential audit 与 UI 状态已实现；Exporter/告警、retention archive、全日志源和 Worker progress/liveness 尚未实现。 +- `delivered`：历史 `0004` / 12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 +- `implemented/pending_gate`：当前工作树的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已实现；后端 `916 passed, 33 skipped`、前端 `38 passed`、真实 PG16/Redis7 `33 passed, 0 skipped`、八规则 promtool 与 dirty capacity/9/9 acceptance 已通过。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。独立 commit、clean-SHA Compose、push 和该精确 SHA 的 CI 尚未完成，因此 P2-06 未完成仓库级收尾。 - `delivered`：`P2-local-control-plane-v2` 在 clean SHA `b6a35fe…` 完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 和 `qualified` 容量模型；只限定记录的 Mock-only 单机拓扑。 - `not_met`：backup/restore 仍未完成；三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 - `delivered`：治理实现 SHA `665244e…` 已 push；本地 enhanced capacity、9/9 acceptance 与远程 run `33099260233` 4/4 均通过。 @@ -233,13 +233,14 @@ ### 交付物 - 已交付基础：PostgreSQL/SQLite migration、Redis/Worker/lease/fencing/recovery、六服务 Compose 与历史故障证据。 -- 已交付候选切片：`0004`、12 表 importer、四层治理/ledger/backpressure/fairness、typed audit/history/Provider/credential evidence、UI 状态、enhanced capacity/PG tests 与运维文档。 +- 已交付候选切片：`0004`、历史 12 表 importer、四层治理/ledger/backpressure/fairness、typed audit/history/Provider/credential evidence、UI 状态、enhanced capacity/PG tests 与运维文档。 +- 已实现待门禁切片：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator 和全日志源治理。 - 治理候选门禁已完成；P2-01 的 v2 多轮资格、实现 commit/push、实现 SHA 4/4 CI 与证据文档精确 SHA 4/4 CI 也已完成，P2-01 仓库级收尾完成。 -- 阶段正式剩余：Exporter/告警、retention archive、Worker progress/liveness、backup/restore 与完整故障矩阵。 +- 阶段正式剩余：先完成 P2-06 的 commit、clean-SHA capacity/acceptance、push、精确 SHA CI 与证据收尾；随后完成 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 ### 状态 -`in_progress`。可靠执行基础、P2-05 治理、P2-06/P2-07 已有切片和 P2-01 v2 单机控制面资格均取得可复核证据与精确 SHA 远程门禁，但 Exporter/告警/retention/backup/Worker-progress 等阶段验收仍缺失。不得把 Phase 2、生产 HA、完整可观测性、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 +`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 功能已在当前工作树实现，但仓库级门禁尚未闭环，P2-07 backup/restore 与完整恢复演练仍缺失。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 ## 6. Phase 3：标准 Benchmark 与代码评测 diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 074838b..b248738 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -15,7 +15,7 @@ LLMBenchLab 当前开发基线是供个人开发者在受信任机器上使用 - Provider API Key、访问令牌、兼容模式环境变量、AES-GCM credential envelope 与部署 keyring。 - Benchmark 题目、参考答案、metadata、许可证与来源信息；未来可能包含私有数据。 - 原始模型回答、解析结果、错误信息、运行配置快照和排行榜证据。 -- Governance policy/scope/minute counters、逐 Provider attempt reservation/settlement ledger、typed audit/history、credential audit 与安全归一化 Provider metadata。 +- Governance policy/scope/minute counters、逐 Provider attempt reservation/settlement ledger、typed audit/history/archive、Worker process/progress、credential audit 与安全归一化 Provider metadata。 - PostgreSQL/SQLite 中的数据库事实、SQLite 源库与备份、Redis AOF/队列元数据、宿主机文件和本地网络访问能力。 - 用户的 Provider 配额、预算与账号信誉。 - GitHub 仓库、Actions 权限、发布产物和依赖供应链。 @@ -68,7 +68,7 @@ LLMBenchLab 当前开发基线是供个人开发者在受信任机器上使用 - 每个 HTTP attempt 的 ledger 状态只允许 `reserved → send_started → settled_actual|settled_conservative`，或可证明未开始外发时 `reserved → released_pre_send`。明确 pre-send release 保留旧 ledger 并开启新 generation，只重试当前未发送 ordinal，不重置已经 `send_started` 的较小 ordinal。commit/usage/transport 不确定一律保守结算；这防止本地预算按零释放，却可能低估剩余额度。 - scope 和 UTC-minute 物化 counter 会在治理变更前从 never-delete ledger 重算；高、低漂移都停止新 admission/结算/对账，并由 API/Worker 边界用独立短事务尽力记录固定、不含损坏值的 `governance_integrity_error`。过期 lease takeover 在旧 ledger 校验失败时撤销新 owner 并使 Run fail closed，防止新 Worker 外发。 - 租约失效后 reconciler 会释放/结算本地 reservation，避免永久占用 admission。该动作无法停止已从 `send_started` 外发、仍在 Provider 运行的幽灵请求，也不证明崩溃后的真实远端并发仍受本地上限约束。本地 consumed 只用于 admission 与审计，必须另行核对 Provider 账单。 -- typed audit payload 只接受 allowlist 短枚举/数值和 opaque ID，不含 Key、Authorization、URL、Prompt、原始回答或异常文本。事件重放以唯一 key 和 payload hash 检查一致性；默认 operational/security 至少保留 90/365 天。它仍是同库的应用 append-only 事实，不是 WORM 或管理员不可篡改证据。 +- typed audit payload 只接受 allowlist 短枚举/数值和 opaque ID，不含 Key、Authorization、URL、Prompt、原始回答或异常文本。事件重放以唯一 key 和 payload hash 检查一致性；默认 operational/security 至少保留 90/365 天。Retention CLI 会把过期完整事实写入权限收紧、canonical、内容/文件 hash 绑定的内部 JSONL，并支持离线 verify、精确 restore/delete/reconcile；hash 不是签名或 WORM，archive 仍含内部身份与时间线，必须按敏感运维文件保护。 ## 3. API Key 与秘密管理 @@ -131,7 +131,8 @@ Web stored 流程如下： 当前应用日志和 OpenAI-compatible Adapter 的错误处理会： -- 对 LLMBenchLab 应用 logger 使用结构化 JSON、请求/Run correlation ID 和字段 allowlist；API 只记录代码定义的 route template，不记录原始查询串或请求体。 +- 只有显式登记的 LLMBenchLab 应用 logger 可输出 literal message 与 structured extra；字段除了 allowlist，还分别受固定 event/result/error/component 枚举、canonical UUID、Redis stream ID、HTTP method/route template 和有限数值合同约束。非法身份字段被省略，未知 method/path/code 只映射到固定 `unsupported`，异常类名只保留固定错误族，不反射原值。API 不记录原始查询串或请求体。 +- `configure_logging` 把已知进程内 `Uvicorn`、`httpx`/`httpcore`、SQLAlchemy 与 Redis client logger 统一路由到同一 sanitizer：第三方动态 logger 名、message、exception 与伪造 structured extra 不进入 JSON；原始 `uvicorn.access` handler 被禁用，由应用 middleware 的 route-template 事件替代。 - 请求校验响应只保留安全的 `type`、白名单化 `loc` 和 `msg`，省略 Pydantic `input`/`ctx`；keyring/加解密失败只暴露稳定错误码。Model create/PATCH 成功响应带 `Cache-Control: no-store`。 - SQLAlchemy engine 固定 `hide_parameters=true`，因此 SQL echo/异常不会打印 bound Key/envelope 参数；这不替代关闭生产 SQL debug、保护数据库和限制第三方 telemetry。 - 用 `[REDACTED]` 替换当前 API Key 的精确值。 @@ -147,8 +148,8 @@ Web stored 流程如下： 仍需遵守以下规则： - 不记录 `os.environ`、HTTP headers、完整异常 locals、请求体或 `.env` 内容。 -- 结构化 JSON/字段 allowlist 只覆盖 LLMBenchLab 应用 logger，不覆盖全部 Uvicorn、access log、SQLAlchemy、Redis client 或其他第三方 logger；部署侧必须另行统一采集、过滤和保留策略。 -- Uvicorn access log 仍可能记录方法、原始路径和状态码；因此秘密不得出现在 URL、查询参数或路径，反向代理 access log 也必须应用同一规则。 +- 应用外的反向代理 access log、PostgreSQL/Redis server log、Docker/runtime log，以及 formatter 配置前或进程崩溃时的 stderr 不受上述 sanitizer 控制；未知且安装自有 handler 的第三方库也必须在部署侧审计、过滤并设置保留策略。 +- 即使进程内原始 Uvicorn access handler 已禁用，秘密仍不得出现在 URL、查询参数或路径；反向代理和基础设施日志必须应用相同规则。 - `DEBUG` 仅用于无秘密的本地排障。HTTP wire logging 和第三方 SDK debug logging 默认关闭；即使 SQL 参数已隐藏，也不应在含敏感数据的部署中采集冗长 SQL trace。 - 原始模型回答可能含供应商回显的其他敏感内容。当前只保证精确替换正在使用的 Key，并识别部分常见秘密形态；回答仍会作为评测证据持久化并通过 Responses API 返回，不能把“没有当前 Key”误当成“可以公开”。 - 异常报告、CI artifact 和截图发出前再次人工检查，不把数据库或 `.env` 整体上传。 @@ -262,32 +263,32 @@ Compose 默认只把 API 和 frontend 端口发布到宿主 `127.0.0.1`，Postgr 健康端点也不是访问控制或完整监控： - `/live` 只证明 API 进程能响应；`/health` 只检查数据库连接；`/ready` 检查数据库、Alembic head 与 Redis，并可能在 Redis 故障时返回 `503/degraded`，同时保留数据库提交和对账能力。 -- `/tasks/metrics` 公开数据库当前任务/governance gauges；`/tasks/history` 在同一数据库读取快照中逐条验证 retained audit 的 contract/hash/identity/retention，再公开 typed event counters 与 Run queue/execution/end-to-end p50/p95/p99，任一损坏事件使整个响应 fail closed 且不反射损坏值。pending/running cancel 都写 `run_cancel_requested`，dead-letter 单列 `run_dead_lettered`；`/runs/{id}/audit` 公开该 Run 的稳定分页 typed event。Run created/finished 与 credential audit 均取数据库 UTC。它们没有租户隔离，history 最多取 10,000 latency 样本并显式 `truncated`，audit 也不是 WORM；无鉴权时这些计数、时序、ledger 关联和 Provider metadata 都是可读取的敏感运维元数据。 -- Worker 容器 probe 只检查数据库/head 和队列依赖能力。Redis 不可用时它会报告 degraded 但以成功退出保留数据库对账路径；它**不证明 Worker 主事件循环仍在领取、心跳或推进任务**，不能替代 watchdog、进度告警或进程级 liveness。 +- `/tasks/metrics` 公开数据库当前任务/governance gauges 与匿名 Worker expected/registered/live/stalled/shortfall、最近聚合进展时间；`/tasks/history` 在同一数据库读取快照中逐条验证 retained audit 的 contract/hash/identity/retention，再公开 typed event counters 与 Run queue/execution/end-to-end p50/p95/p99。`/metrics/prometheus` 以固定 labels 输出同类 current/window/latency/Worker gauges，并用硬行数上限与 per-process single-flight 约束抓取压力。任一损坏 retained event 使整个 history/exporter fail closed 且不反射损坏值。pending/running cancel 都写 `run_cancel_requested`，dead-letter 单列 `run_dead_lettered`；`/runs/{id}/audit` 公开该 Run 的稳定分页 typed event。它们没有认证或租户隔离；无鉴权时这些计数、时序、ledger 关联和 Provider metadata 都是可读取的敏感运维元数据。 +- Worker 主进程只把固定 scan/claim/progress/lease-heartbeat bit 合并写入 DB UTC `worker_processes`，API/exporter不返回 generation/worker ID。无真实 event 时 recorder 不刷新 `last_seen_at`，避免 timer 制造幽灵健康；异常退出的 generation 会保守变 stale。Worker 容器 probe 仍只检查数据库/head 和队列依赖能力并明确 `main_loop_progress=not_checked`；它**不证明 Worker 主事件循环正在推进**，不能替代 DB-time progress 告警。 - API readiness 把同步数据库检查放入 `asyncio.to_thread` 并限制等待时间；asyncio 超时不会取消已进入线程的数据库驱动调用，实际资源占用仍由数据库连接/驱动/池 timeout 约束。不得把 HTTP 探针 timeout 当作数据库查询硬中止。 -Redis 仅可置于受控内部网络。当前本地 Compose 使用 AOF、无 ACL/TLS，不能暴露公网或共享开发网；更高信任级别部署必须启用 Redis ACL/认证、TLS、最小网络可达、磁盘权限与备份策略，并限制危险管理命令。Redis 消息只含内部 ID、版本与 correlation ID，不含 Prompt、答案、Provider Key 或权威 Run 状态；清空、丢失或重复消息应只影响延迟/可用性，数据库事实不能被 Redis 覆盖。 +Redis 仅可置于受控内部网络。当前本地 Compose 使用 AOF、无 ACL/TLS，不能暴露公网或共享开发网；更高信任级别部署必须启用 Redis ACL/认证、TLS、最小网络可达、磁盘权限与备份策略，并限制危险管理命令。Redis 消息只含 canonical UUID Run/correlation ID 与固定版本，不含 Prompt、答案、Provider Key 或权威 Run 状态；publish 和 delivery 两侧都拒绝非 UUID identity，防止受污染通知在数据库校验前进入 Worker structured log。清空、丢失或重复消息应只影响延迟/可用性，数据库事实不能被 Redis 覆盖。 ## 9. 数据库、响应、迁移与备份 - PostgreSQL/SQLite、Redis AOF、备份和容器 volume 整体都没有存储层加密；只有 `model_credentials` 的 Provider Key 字段在应用层形成 AES-GCM envelope。题目、回答、endpoint、环境变量名和其他元数据仍是明文，应使用专用系统账号和最小文件权限保护，不放在云盘公共共享目录。Compose 中的 `llmbenchlab-local-only` PostgreSQL 密码只是隔离本地开发占位，不是生产秘密。 - 数据库备份可能比在线数据库保留更久，应应用同等或更严格的访问、加密、保留和销毁策略。部署 keyring 必须单独备份：不备份会失去恢复能力，与数据库备份放在一起则失去 envelope 的隔离价值。 - Responses API 包含 raw response、Prompt、标准答案和错误；Questions API 包含参考答案和 metadata。Web Run Detail 每次分页读取 100 条只限制单次展示量，不是授权、脱敏或数据隔离；Runs 列表中的详情链接也不会改变任何人的读取权限。不要把这些接口接入公开 Dashboard。 -- Governance policy、四层 opaque scope/Token/费用聚合、逐 attempt ledger、typed audit/history 和 Provider request/model/fingerprint/finish metadata 虽不含凭据正文，仍可暴露调用规模、费用趋势、供应商关联和故障时间线，必须与 Run 证据同级保护。应用 append-only 与 payload hash 不阻止数据库管理员修改或删除这些行。 +- Governance policy、四层 opaque scope/Token/费用聚合、逐 attempt ledger、typed audit/history/archive、Worker progress 和 Provider request/model/fingerprint/finish metadata 虽不含凭据正文，仍可暴露调用规模、费用趋势、内部身份关联和故障时间线，必须与 Run 证据同级保护。应用 append-only 与 payload/file hash 不阻止数据库管理员或本地文件所有者修改、替换或删除事实。 - 正式评测报告同样包含题目、参考答案、raw response、解析和错误证据。Exporter 创建权限收紧的目录/文件、拒绝覆盖已有目标并脱敏常见秘密形态；报告指标从计划题与 Responses 重算，`metrics_provenance` 会显式标注 Run 汇总字段漂移。操作者仍须像保护数据库一样保护 `summary.json`、`groups.csv` 与 `responses.jsonl`；脱敏不是内容访问控制。 - Run 快照用于复现，会保存模型/Benchmark 标识、`credential_source`、Model ID、endpoint，以及 environment 模式的变量名称；不会保存 credential row ID、ciphertext、nonce、key id 或 plaintext。设计环境变量名时避免包含机密业务信息。 - 删除 Model 会在存在历史 Run 时被拒绝，以保护证据；MVP 尚无合规删除、匿名化或数据生命周期功能。 - 备份与恢复步骤见 [DEPLOYMENT.md](DEPLOYMENT.md)。 -显式 SQLite→PostgreSQL importer 会按依赖顺序复制 12 张核心/治理表的**完整内容**：`governance_policies`、`models`、`model_credentials`、`benchmarks`、`questions`、`governance_scopes`、`evaluation_runs`、`evaluation_responses`、`governance_minute_buckets`、`question_executions`、`provider_call_reservations`、`audit_events`。因此它会复制 encrypted credential envelope、题目/答案/快照/原始回答、typed Provider metadata、policy、Token/费用 ledger 与 audit，但不会解密或打印 row 内容。它只能在受信环境中针对停止写入、已在 Alembic head 且无 active reservation 的源库和空目标执行；在打开目标前还会从全部 ledger 重算 scope/minute 计数，任何高/低漂移或缺 bucket 都拒绝导入。源库、目标库、对账输出与中间备份必须按最高敏感数据等级保护。工具输出只包含每表行数、主键集 SHA-256 digest 和 canonical row SHA-256 digest；摘要仍可能用于关联同一快照，不等于加密或访问控制。 +显式 SQLite→PostgreSQL importer 会按依赖顺序复制 13 张核心/治理表的**完整内容**：`governance_policies`、`models`、`model_credentials`、`benchmarks`、`questions`、`governance_scopes`、`evaluation_runs`、`evaluation_responses`、`governance_minute_buckets`、`question_executions`、`provider_call_reservations`、`audit_events`、`worker_processes`。因此它会复制 encrypted credential envelope、题目/答案/快照/原始回答、typed Provider metadata、policy、Token/费用 ledger、audit 与 stopped/stale Worker facts，但不会解密或打印 row 内容。它只能在受信环境中针对停止写入、已在 Alembic head 且无 active reservation/live Worker generation 的源库和空目标执行；在打开目标前还会从全部 ledger 重算 scope/minute 计数，任何高/低漂移或缺 bucket 都拒绝导入。源库、目标库、对账输出与中间备份必须按最高敏感数据等级保护。工具输出只包含每表行数、主键集 SHA-256 digest 和 canonical row SHA-256 digest；摘要仍可能用于关联同一快照，不等于加密或访问控制。 - 含凭据的目标 DSN 必须通过 `--target-env ENV_VAR`（默认 `LLMBENCHLAB_DATABASE_URL`）读取；`--target` 只接受无密码 URL。仍需防止环境、进程转储和 CI 配置泄露 DSN。 - 退出码 `0` 表示提交后对账成功；退出码 `2` 表示提交前失败并回滚目标事务。 -- 退出码 `4` 表示 PostgreSQL 未确认 `COMMIT` 结果：原子事务保证目标应为“空”或“完整”，但客户端不知道是哪一种。**禁止盲目重试**，必须先检查目标 12 表、Alembic head 和已有对账证据。 +- 退出码 `4` 表示 PostgreSQL 未确认 `COMMIT` 结果：原子事务保证目标应为“空”或“完整”，但客户端不知道是哪一种。**禁止盲目重试**，必须先检查目标 13 表、Alembic head 和已有对账证据。 - 退出码 `3` 表示 `COMMIT` 已确认、但提交后验证或报告失败；导入已经提交。**禁止盲目重试或把它描述为回滚**，应先只读核验目标，必要时从已验证备份执行人工恢复。 - 工具是单向导入，不提供 PostgreSQL→SQLite 自动回迁。schema downgrade 也不等于数据平台回滚。 -`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata，guard 因而在第一条 DDL 前拒绝任何相关表行或新 Run/Response 事实。正常 bootstrap/运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的 pre-0004 备份并保留 post-0004 只读 ledger/audit 归档。只有隔离空数据库才用于 `0004 → 0003 → 0004` 往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 +`20260828_0005 → 0004` 会移除 Worker progress，guard 在第一条 DDL 前拒绝任何 generation 行；`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata并拒绝任何相关事实。正常运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的旧备份并保留只读证据。只有隔离空数据库用于双方向往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 ## 10. 依赖与构建供应链 @@ -332,7 +333,7 @@ Mock CI 不需要任何 Provider Key，也禁止配置真实模型凭据。标 - 身份认证、对象级授权、管理员权限、安全会话/API Token，以及把现有 typed audit 送入受控归档/WORM 或等价独立安全日志。 - SSRF 应用层校验与网络层出站 allowlist。 - 将现有共享 DB governance 扩展为认证主体/租户 policy、Provider 熔断和生产级 Worker 隔离；在目标硬件与精确提交重跑真实工作负载容量/HA 验证，不能把当前 Mock dirty-worktree 基线当 SLA。 -- 将现有 typed counters/p50/p95/p99 接入 exporter、监控面板和告警，补 Worker 主循环 liveness、不可抵赖审计选项与经过演练的多主机恢复；当前 Runbook/Mock 故障演练不等于生产 HA。 +- 将现有 exporter/固定告警接入受控 Prometheus/Alertmanager、认证 Dashboard 与组织值班流程，并另行设计密码学签名/WORM 或等价不可抵赖审计；当前 DB-time Worker progress、普通文件 archive 和 Mock 故障演练不等于生产 HA、不可篡改存储或多主机恢复。 - TLS、反向代理请求体限制、安全响应头、CSP、Host/代理信任配置。 - 私有题目、参考答案、raw response 的访问控制、静态加密、保留与删除策略。 - 集中 secrets manager、Key 轮换、备份加密与恢复演练。 diff --git a/docs/TESTING.md b/docs/TESTING.md index 6f3a22c..0787a57 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -13,12 +13,13 @@ LLMBenchLab 的默认验收路径必须完全离线、可重复且不产生模 | 后端单元测试 | Evaluator、Adapter、Loader/Hash、固定标准数据转换的边界语义 | `backend/tests/test_evaluators.py`、`test_adapters.py`、`test_dataset_loader.py`、`test_standard_datasets.py` | 禁止；Provider 用 MockTransport，下载用注入 fixture fetcher | | 正式流程组件测试 | CLI 秘密/确认/编排、模型发现/canary、完整报告和大题集有界 task | `test_evaluation_cli.py`、`test_provider_preflight.py`、`test_run_report.py`、`test_evaluation_runner_reliability.py` | 禁止；只用 fixture、MockTransport、Mock Adapter、临时数据库 | | API 与进程边界测试 | FastAPI Schema、状态码、秘密安全、Run 提交与 API/Worker 分离 | `backend/tests/test_api.py`、`test_run_dispatch.py`、`test_process_boundaries.py` | 禁止 | -| Governance / audit 测试 | full-document policy/hash/Run overrides、四层 admission/ledger 派生物完整性、fail-closed Token/cost、pre-send generation、公平 slice、typed history/audit、Provider metadata 与 credential audit | `test_governance.py`、`test_governance_api.py`、`test_audit_api.py`、`test_task_history_api.py`、`test_response_metadata_api.py`、`test_credential_audit.py` | 禁止；SQLite/Mock/MockTransport | -| 租约与 Worker 测试 | 条件领取、fencing、心跳、取消、幂等 Response、重试/恢复、队列 ACK | `test_run_leases.py`、`test_evaluation_runner_reliability.py`、`test_worker.py`、`test_task_queue.py` | 禁止；SQLite/假队列 | -| 迁移与导入回归 | SQLite/真实 PostgreSQL migration、0004 populated downgrade refusal/空库往返，以及 12 表 SQLite→PostgreSQL 原子导入 | `test_migrations.py`、`test_sqlite_postgres_import.py` | 导入/本地部分禁止；真实 PostgreSQL 用 `integration` marker | +| Governance / audit 测试 | policy/ledger 完整性、typed history/audit、Provider/credential evidence，以及 canonical archive/离线 verify/精确 reconcile/restore/delete | `test_governance.py`、`test_audit_api.py`、`test_audit_archive.py`、`test_audit_retention.py`、`test_audit_retention_cli.py` | 禁止；SQLite/Mock/fixture；真实 PG retention 只连测试库 | +| 租约与 Worker 测试 | 条件领取、fencing、心跳、取消、幂等 Response、重试/恢复、队列 ACK，以及 generation 级 DB-time scan/claim/lease-heartbeat/progress/stale | `test_run_leases.py`、`test_evaluation_runner_reliability.py`、`test_worker.py`、`test_worker_progress.py`、`test_worker_probe.py` | 禁止；SQLite/假队列 | +| Metrics / alert / logging 测试 | 固定 Prometheus exposition、snapshot/hard cap/single-flight/取消竞态、精确八规则/Runbook，以及全部生产 logger source/第三方 handler 治理 | `test_prometheus_exporter.py`、`test_prometheus_alert_rules.py`、`test_logging.py`、`test_logging_sources.py` | 禁止；SQLite/假队列/标准库 JSON | +| 迁移与导入回归 | SQLite/真实 PostgreSQL migration、`0005` populated downgrade refusal/空库往返，以及 13 表 SQLite→PostgreSQL 原子导入 | `test_migrations.py`、`test_sqlite_postgres_import.py` | 导入/本地部分禁止；真实 PostgreSQL 用 `integration` marker | | 真实基础设施集成 | PostgreSQL 并发领取/取消竞态、Redis Streams PEL/ACK/重复投递 | `backend/tests/integration/` 与 importer 的 `integration` 用例 | 只连接显式测试 PostgreSQL/Redis；禁止 Provider | | Mock 端到端 Smoke | API 提交 pending Run → 独立 WorkerService → Responses → Leaderboard/Metrics | `backend/tests/test_smoke.py`，marker 为 `smoke` | 禁止 | -| Compose 故障验收 | 六服务拓扑、双 Worker、进程/Redis/lease 故障、治理/ledger、取消、重复消息与 0004 安全回滚 | `scripts/phase2_acceptance.py` / `make phase2-acceptance` | 只拉取/构建基础镜像；模型执行始终为离线 Mock | +| Compose 故障验收 | 六服务拓扑、双 Worker、进程/Redis/lease 故障、治理/ledger、取消、重复消息、Worker expected count 与 `0005` 安全回滚 | `scripts/phase2_acceptance.py` / `make phase2-acceptance` | 只拉取/构建基础镜像；模型执行始终为离线 Mock | | Mock 容量基线 | 真实 PostgreSQL 16/Redis 7、全有限 policy、1/2 Worker、精确 `202/429` backlog、cooperative quantum、跨 Model 公平性、lease/Redis/重复通知故障及 DB/queue/ledger/audit 对账 | `scripts/phase2_capacity.py` / `make phase2-capacity` | 只拉取/构建基础镜像；模型执行始终为离线 Mock | | 固定单机资格 | clean SHA 上的 v2 四-cell、1 warm-up + 5 measured、统计/恢复/公平/ledger/cleanup 硬门禁 | `scripts/phase2_slo.py` / `make phase2-slo` | 只拉取/构建基础镜像；模型执行始终为离线 Mock | | 前端单元/组件测试 | 格式化、状态/指标、错误/空态、主要页面交互 | `frontend/src/**/*.test.ts(x)`、`frontend/tests/` | API 必须 stub/mock | @@ -26,7 +27,7 @@ LLMBenchLab 的默认验收路径必须完全离线、可重复且不产生模 | 构建检查 | 确认生产前端可编译打包 | `npm run build` | 安装完成后不需要 | | 配置检查 | 校验 Compose 插值和服务定义 | `docker compose config` | 不启动 Provider | -单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有自动化覆盖；增强 capacity、9/9 acceptance 已通过，`P2-local-control-plane-v2` 也已在 clean SHA `b6a35fe…` 完成 1+5、23/23 和远程 4/4 门禁。Phase 2 仍为 `in_progress`，因为这项 Mock-only 单机资格不能替代 Worker progress/liveness、告警/归档、backup/restore、剩余恢复矩阵、生产 SLA 或 HA 证明。 +单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有已交付自动化；P2-06 当前又加入 Worker progress、exporter/规则、retention 与全日志源回归。合并定向、lint、全量 test、smoke、真实 integration、隔离 migration、build、Compose config、Prometheus 规则解析、dirty 工作树 capacity 与 9/9 acceptance 已全绿；structured-extra High 与 Worker `__main__` logger Medium 修复后，最新 76-file staged 技术/安全终审为 0 Blocker/High/Medium。当前仍待 clean-SHA Compose 证据与精确 SHA CI，不能把已通过的本地层级写成 P2-06 仓库级收尾。Phase 2 还缺 P2-07 backup/restore、剩余恢复矩阵、生产 SLA/HA 证明，因此继续 `in_progress`。 ## 3. 环境准备 @@ -222,6 +223,7 @@ OpenAI-compatible 测试只给进程内 transport 使用虚构 token；不得把 - backlog 满稳定 429；rate/concurrency defer 不生成 0 分 Response；每个 lease 只新增 frozen question quantum，cooperative yield 不增加 `failed_attempt_count`，最久未获服务 Run 优先再调度。 - typed audit 的唯一 event key/payload hash、90/365-day retention、稳定 Run audit 分页，以及 history window/p50/p95/p99 截断语义；history 必须在单一一致性快照中校验窗口内每个 event 的 contract、payload hash、identity、retention 和数值边界，任一损坏使整个响应以 `audit_event_integrity_error` fail closed，不能返回局部统计或反射损坏 payload。credential change/reject/decrypt failure 不含 Key/origin/envelope，Provider metadata 经过安全归一化进入 Response/API/report。 - Worker 每次只执行一个 Run，按配置心跳；优雅停止在 grace 内等待，超时后由租约过期恢复，而不是伪造成功 ACK。 +- 长运行 Worker 在执行前注册唯一 process generation；真实 scan/claim/lease-heartbeat/progress 只设置固定 bit，并由单 in-flight recorder 以同一个 DB timestamp 节流写入。无 event 时零写入、flush 失败保留 bit、graceful stop 原子保存 pending/stop，stopped generation 的迟到 flush 不能复活；`last_seen_at == cutoff` 明确为 live。 - Runner 对大题集只创建至多 `concurrency` 个消费者 task；当前可靠性用例以 2,000 题验证并发 4 的固定 task 集，并验证取消/失租停止后续取题和 Adapter 只关闭一次。另有阻塞快照物化用例验证同步加载被移出事件循环，租约心跳可在加载期间继续运行。 - API 进程不持有 Runner/task manager；只启动 API 时 Run 保持 `pending`，启动独立 Worker 后才执行。 @@ -239,6 +241,8 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 - Redis 不可用时 `/ready` 返回脱敏的 `503 degraded`，但数据库正常时 `accepting_runs=true`、database reconciliation 可用；数据库或 schema 不可用时返回 `not_ready` 并停止接受新 Run。 - 服务端生成并回传 UUID `X-Request-ID`，忽略客户端同名值，防止调用方把 write-only Key 复制到 header 后迫使日志/响应反射；未知路径只记录 ``，不把用户路径或请求正文写入应用日志。 - `/tasks/metrics` 从数据库派生任务、governance backlog/delay/exhaustion、active Provider attempt、overdrawn scope 与 attempt/failure/dispatch gauges；`/tasks/history?window_hours=1..2160` 从 retained typed audit/Run 时间聚合 counters 与 queue/execution/end-to-end p50/p95/p99，并验证 10,000 样本上限/`truncated`。history 的 DB 时钟、窗口、事件和三组 latency 查询必须共享同一事务快照（PostgreSQL `REPEATABLE READ READ ONLY`；SQLite 显式 `BEGIN`）；任一 event contract/hash/identity/retention/数值损坏都使整个响应以 500 `audit_event_integrity_error` fail closed。pending/running cancel 都贡献 `run_cancel_requested`，dead-letter 使用专用 `run_dead_lettered`，不会与一般 terminal counter 混淆。 +- `/tasks/metrics` 另验证 expected/registered/live/stalled/shortfall、inclusive stale cutoff 与最近五类 Worker activity 时间不泄露 process ID；`/metrics/prometheus` 固定 content type、LF/final newline、HELP/TYPE/sample 顺序、全部 gauge/enum label、无动态查询参数和 `Cache-Control: no-store`。 +- exporter 回归验证单 DB-time snapshot、audit `50,001` hard fail、每类 latency `10,001` truncated、数据库/audit/renderer 整次 fail closed、Redis 非权威降级、进程内 single-flight，以及重复 request cancellation 时 DB thread 完成前 gate 不提前释放；任何响应都不得反射损坏 payload、异常文本、对象 ID 或秘密 marker。 - governance policy GET 在初始化前无副作用返回 `404 governance_policy_not_initialized`，PUT 必须提交全部 20 个字段；相同内容幂等、历史内容重激活原 ID/version。整数限制覆盖各字段准确边界；所有 USD 限制公开上限均为 `10000000.00000000`，请求接受 Decimal-compatible JSON number 或 decimal string，响应固定为 JSON string。SQLite 回归验证该上限内 binary64 spacing 小于半个 `0.00000001` 量子，八位小数可按相同 Decimal round-trip；PostgreSQL 仍以 `NUMERIC(20,8)` 为并发生产门禁。Run audit 端点按 `(occurred_at,id)` 稳定分页并对 retained event identity/payload/retention 做读取期完整性检查。 - Model CRUD、分页、Provider 必需字段、远端 HTTP 拒绝/loopback 例外和名称冲突。 - Model POST/PATCH 接受 8–8192-byte visible-ASCII write-only `api_key`，并拒绝 7-byte、空白、非 ASCII 与过长输入；GET/list 的凭据相关字段只返回非秘密状态，不返回 Key、密文、nonce 或 key id。marker 测试覆盖 201、422、409、503、500、Host 与 request-ID 反射路径。 @@ -258,7 +262,7 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 `backend/tests/test_migrations.py` 使用独立临时 SQLite 和 Alembic 子进程验证： -- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260827_0004` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata 字段、约束和索引。 +- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260828_0005` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata、`worker_processes` 字段/约束，以及两个 bounded audit scan indexes。 - 有模型、Benchmark、题目、Run 与 Response 的 legacy schema 被一致性备份、严格识别并无损升级；题目按原插入顺序回填 0-based `position`。 - 与当前 metadata 一致但没有版本标记的库可安全收养，已有 head 重复 preflight 不生成多余备份。 - 部分表、server default/CHECK 内容或重名、PK/UNIQUE/FK/index/partial index、trigger、SQLite conflict policy/generated column、`STRICT`/`WITHOUT ROWID` 等未知 drift 在创建版本标记和备份前被拒绝；已在 head 的库同样验证。 @@ -266,6 +270,7 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 - `0001 -> 0002` 会按冻结的 Phase 1 语义收敛旧 `running` Run；存在 active Run 时可靠性 downgrade 被拒绝，不能静默删除租约元数据。 - `0002 -> 0003` 会把旧 OpenAI-compatible Model 回填为 `environment`、Mock 回填为 `none`；只要凭据表有任意行，credential downgrade 在 DDL 前拒绝并保留二进制内容。 - `0003 -> 0004` 把既有 Run 标为 `legacy_unmanaged` 并保留 protocol-v1 证据；任意 policy/scope/bucket/question execution/attempt ledger/audit、新 Run fairness/governance 字段或 Response Provider metadata 存在时，`0004 -> 0003` 必须在第一条 DDL 前拒绝。只有隔离空库用于 `0004 -> 0003 -> 0004` roundtrip。 +- `0004 -> 0005` 不回填虚构 Worker generation；任意 `worker_processes` 行都使 `0005 -> 0004` 在第一条 DDL 前拒绝。只有显式清空 process facts 或隔离空库才能往返，进入 `0004` 后原 governance/audit downgrade guard 继续生效。 - 应用启动 revision 门禁拒绝未迁移库；测试夹具中的 `create_all` 仅用于隔离临时库，并显式 stamp 到与 metadata 对应的 head，不是运行时建表路径。 目标化运行： @@ -275,13 +280,13 @@ cd backend uv run pytest tests/test_migrations.py ``` -真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline 后停止 API/Worker，尝试 `0004 -> 0003` 并断言 guard 在任何 DDL 前拒绝，revision 和 core protocol hash 不变；另建隔离空 PostgreSQL 验证 `0004 -> 0003 -> 0004`。schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 +真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline 并停止 API/Worker 后，先断言 populated `0005 -> 0004` 在任何 DDL 前拒绝且 head 不变；另建隔离空 PostgreSQL 完成 `0005 -> 0004 -> 0005`/check。历史 `0004 -> 0003` governance/audit guard 仍保留；schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 ### 6.3 SQLite→PostgreSQL 导入 -`backend/tests/test_sqlite_postgres_import.py` 的离线路径验证 canonical hash、只读 SQLite、head/integrity/FK/active-Run/active-reservation 拒绝、固定 12 表复制及提交前回滚；源 preflight 在打开目标前从完整 reservation ledger 重算每个 scope 和 minute bucket，拒绝高报、低报、缺 bucket 或其他派生漂移。fixture 含真实 AES-GCM nonce/ciphertext/key-id、governance/ledger/audit/Provider metadata 行并断言明文不在 SQLite。标记为 `integration` 的真实 PostgreSQL 用例还验证： +`backend/tests/test_sqlite_postgres_import.py` 的离线路径验证 canonical hash、只读 SQLite、head/integrity/FK/active-Run/active-reservation/live-Worker 拒绝、固定 13 表复制及提交前回滚；源 preflight 在打开目标前从完整 reservation ledger 重算每个 scope 和 minute bucket，拒绝高报、低报、缺 bucket 或其他派生漂移。fixture 含真实 AES-GCM nonce/ciphertext/key-id、governance/ledger/audit/Provider metadata 与 stopped/stale Worker rows，并断言明文不在 SQLite。标记为 `integration` 的真实 PostgreSQL 用例还验证： -- 随机专用空库成功导入后，12 表行数、主键集合和 canonical row hash 与源一致，JSON、Decimal、UTC、协议快照、逐题证据、governance ledger/audit 及 credential 二进制保持；stdout/stderr 不打印 Key、key id、nonce 或 ciphertext。 +- 随机专用空库成功导入后，13 表行数、主键集合和 canonical row hash 与源一致，JSON、Decimal、UTC、协议快照、逐题证据、governance ledger/audit、Worker progress 及 credential 二进制保持；stdout/stderr 不打印 Key、key id、nonce 或 ciphertext。 - 中途复制失败整体回滚；两个不同源并发导入时恰好一个成功，另一个在目标非空检查处拒绝。 - `COMMIT` 确认丢失使用专用 `commit_outcome_unknown` 语义；已确认提交后的 snapshot 或输出失败使用 `committed_but_verification_failed`，两者都禁止盲目重试。 - 源 SQLite 主文件 hash 不变；输出只有阶段、表名、行数与 SHA-256 摘要，不打印题目、回答或连接 URL。 @@ -292,9 +297,11 @@ uv run pytest tests/test_migrations.py ### 6.4 健康、日志和指标的测试边界 - `/ready` 把同步数据库/head 检查放入 `asyncio.to_thread` 并设置异步等待上限，Redis ping 也有独立 timeout；测试证明半开依赖不会阻塞 `/live`。取消 `to_thread` 的等待不会终止底层数据库驱动调用，因此最终上界仍依赖 driver、连接池和 `connect_timeout`，不能把 asyncio timeout 当作强制中止。 -- Worker `app.worker_probe` 是依赖能力探针：数据库/head 失败 exit 1；Redis 不可用但数据库 reconciliation 可用时输出 `degraded` 且 exit 0；配置错误 exit 1。它不观察 Worker 主事件循环或当前 heartbeat，不能证明进程没有卡死。 -- LLMBenchLab 应用 logger 输出脱敏 JSON、request/correlation ID 与 allowlist 字段；异常只记录类型。Uvicorn 自身及 access logger 仍使用其原生 handler，不在“全部日志统一 JSON”的保证内。 -- `/tasks/metrics` 是当下数据库 gauges；`/tasks/history` 和 `/runs/{id}/audit` 已提供 retained typed counters、有限样本延迟与稳定分页事件，包括 pending cancel 的 `run_cancel_requested` 和专用 `run_dead_lettered`。它们仍不是 Prometheus exporter、trace、自动告警、WORM/不可抵赖审计或 Worker 主循环 liveness；这些生产观测能力仍需后续工作。 +- Worker `app.worker_probe` 是依赖能力探针：数据库/head 失败 exit 1；Redis 不可用但数据库 reconciliation 可用时输出 `degraded` 且 exit 0；配置错误 exit 1。它固定声明 `probe_scope=dependencies_only`、`main_loop_progress=not_checked`；主循环进展由独立 DB-time generation facts 聚合，probe 不能读取 peer aggregate 后冒充当前进程健康。 +- 全部生产 logger source 必须使用无格式参数字面量消息；LLMBenchLab JSON formatter 只输出 allowlist extra、把非有限值变为 `null`、异常只保留类型。外部 logger 的动态 message/identity 被固定分类替代，raw Uvicorn access handler 关闭；AST/source 测试阻止新增绕过。 +- `/tasks/metrics`、`/tasks/history`、`/runs/{id}/audit` 与固定 `/metrics/prometheus` 已实现；八条规则由标准库 strict JSON 测试精确校验，并已用临时 `prom/prometheus:v3.5.0` 容器内 `promtool check rules` 验证八条规则全部成功。仓库仍没有 trace、Alertmanager/sender、WORM/不可抵赖审计或自动请求链路删除。 + +Audit retention 目标回归另要求：canonical bytes/hash/rollup/冻结 fixture、128 MiB/64 KiB/10,000 限制、在逐行 decode 前执行全局行数上限、duplicate keys/non-finite/乱序/冲突、owner/`0600`/symlink/parent 权限、FIFO/非普通文件拒绝与 no-replace atomic output；fresh process `verify` 在无效 DSN 下仍成功且不创建 engine/数据库目录；SQLite 和真实 PostgreSQL 完成 archive→verify→delete→reconcile→restore，PostgreSQL mutation 保持 advisory/row lock，零行 no-op 也必须 post-verify，commit unknown/post-commit verification 用独立 exit `4`/`3`，且所有错误不反射 archive path、argv marker、DSN 或行内容。 ## 7. 完全离线 Smoke Test @@ -345,7 +352,7 @@ Smoke Test 证明 API 与 Worker 责任边界以及数据库驱动的最小离 真实模型验收应在隔离的本地数据库上先运行 `--limit`，人工核对模型发现、付费 canary、请求上界、逐题错误、Provider 账单和报告三文件，再决定是否 `--full`。这项手工操作不得写入自动化测试结果或 CI 通过数；如果本次没有真实 API URL/Key，就应明确记录“未运行”，不能用 MockTransport 结果替代“真实 Provider 已验证”。 -当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider request ID/returned model/system fingerprint/finish reason/HTTP attempt count 的持久化/API/报告路径，以及非法值归一化。已知观测缺口是 `resume` canary 尚未追加独立 audit event，另无 exporter/告警/WORM；不能把这些剩余边界写成 Provider exactly-once 或生产可观测性完成。 +当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider metadata、Worker progress、固定 exporter/八规则和 canonical audit retention；合并定向套件、最新 dirty capacity 与 9/9 acceptance 全绿，真实 PostgreSQL/Redis integration 为 `33 passed, 0 skipped`。clean-SHA Compose 和精确 SHA CI 仍待，已知观测缺口还包括 `resume` canary 未追加独立 audit event、仓库不部署 sender/trace/WORM；不能把这些边界写成 Provider exactly-once、生产监控平台或 Phase 2 完成。 ## 9. 前端测试要求 @@ -374,12 +381,32 @@ GitHub Actions 对 `main` push 和 Pull Request 触发四类 job： | Job | 必需检查 | 隔离与失败规则 | | --- | --- | --- | | `backend` | Ruff lint/format；临时 SQLite `upgrade -> 0001 -> head`/check；`pytest -m "not integration"` | 临时 SQLite；不启动 PostgreSQL/Redis；离线 Mock/MockTransport | -| `backend-integration` | 真实 PostgreSQL migration 往返；PostgreSQL/Redis/importer 的 29 个 `integration` 用例 | Actions service 容器；JUnit 必须收集非零用例且零 skip，否则 job 失败 | +| `backend-integration` | 真实 PostgreSQL migration 往返；PostgreSQL/Redis/importer/audit-retention 的 31 个 `integration` 用例 | Actions service 容器；JUnit 必须收集非零用例且零 skip，否则 job 失败 | | `full-stack-reliability` | `python3 scripts/phase2_acceptance.py` 的隔离 Compose 九场景 | 唯一项目/卷、随机 loopback 端口、Mock-only；总是上传已脱敏 evidence，脚本总是精确清理 | | `frontend` | ESLint、Vitest 组件测试、production build（`tsc -b` + Vite） | `npm ci` 锁定依赖；fetch/Recharts stub；`VITE_API_BASE_URL=/api/v1` | CI 不配置 Provider Key、不调用真实模型，也不在线下载 MMLU-Pro/GPQA。PostgreSQL/Redis 是测试依赖，不是 Provider 网络；标准数据转换只使用 fixture fetcher。`P2-local-control-plane-v2` 的 validator、统计、四-cell 编排失败路径、ledger projection 和 exact-project cleanup 合同可以进入普通自动化，但 GitHub-hosted runner 不运行 `make phase2-slo` 的绝对吞吐/延迟门禁：共享 runner 的 CPU、内存和 Docker 调度不是稳定性能实验室。正式实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功；它是同一 SHA 的正确性门禁，不替代本机 1+5。所有必需 job 通过后才能合并；跳过用例、降低断言或使用 `continue-on-error` 都不算修复。具体分支和 Review 门槛见 [GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md)。 +P2-06 当前工作树的阶段性证据如下；尚未完成项必须保持 pending，不能沿用任何历史 SHA 的绿色结果： + +| 验证 | 当前实际结果 | +| --- | --- | +| 合并定向套件 | Worker progress/API/runner/migration/importer、exporter/rules、audit archive/retention/CLI 与 logging 回归全绿；包含 importer committed-target integrity、retention no-op postverify、外部 logger extra、FIFO/line-cap 和 PostgreSQL advisory/row-lock 终审回归 | +| 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped`；包含新增 PostgreSQL audit retention round-trip、advisory/row-lock 门禁与既有 lease/governance/importer 路径。首次 cleanup 命令被本地安全策略拒绝，容器尚未启动；改用明确目标后完整流程通过 | +| `make lint` | 全绿：Ruff 检查 152 files、Ruff format check、ESLint 与 TypeScript typecheck 通过 | +| `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed`；模型路径只使用 Mock/MockTransport/stub | +| `make smoke` | `1 passed, 7 deselected`；完全离线 Mock | +| Frontend production build | 从根目录误运行 `npm run build` 因无 `package.json` 失败；改为 `cd frontend && npm run build` 后成功（2192 modules），保留 662.39 kB 主 chunk 非阻断 warning | +| Migration | 默认用户 SQLite 尚未在 head，直接 `alembic check` 失败且未擅自迁移；临时 SQLite head→`0001`→head/check 与隔离真实 PostgreSQL 往返/check 全绿；当前 head `20260828_0005` | +| Compose config | `docker compose config --quiet` exit 0 | +| Prometheus rules | 临时 `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | +| Ruff scripts 补充检查 | 对 `scripts/` 的过宽默认 Ruff 命令报告 93 条既有 modernization 告警；按本任务合同使用 `--select E,F,I` 后通过，未把既有告警冒充本次失败 | +| Dirty Compose acceptance | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和容器/卷/网络空清理均通过 | +| Dirty Compose capacity | 最新脚本通过；evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`，最终 18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1229 audit，0 error/drift/duplicate/PEL/lag，expected Worker=2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | +| Staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 修复目标集 `67 passed` | +| 仍待门禁 | implementation commit、clean-SHA Compose、push 与精确 SHA CI 尚待完成 | +| Clean-SHA Compose / remote | 尚无 P2-06 独立 commit、clean-SHA capacity/acceptance、push、精确 SHA 或 Actions URL；P2-06 仓库级收尾保持 `in_progress` | + 下表是 2026-08-25 Phase 2 可靠性切片的历史基线，不是本次标准数据/真实评测 CLI 提交的通过声明。本次实际数量必须在当前工作日志和精确 commit 的 CI 中重新记录，不能沿用这些数字： | 验证 | 2026-08-25 历史基线 | @@ -543,7 +570,7 @@ make phase2-acceptance 6. Redis 完全 stop/start；`live`/`health` 保持可用、`ready` 降级，API 仍以 `202` 提交数据库事实，Worker 仅靠 DB reconciliation 完成；Redis 恢复后新消息正常 ACK。 7. Worker 停止时取消 pending Run；Worker 恢复消费旧通知后终态和 0 Response 不漂移。 8. 运行中取消并再次 XADD 同一 Run；Response 数在取消后冻结，重复投递被 ACK 且 canonical snapshot 不变。 -9. 停止 API/Worker 后在 populated PostgreSQL 上尝试 `20260827_0004 -> 0003`：policy/ledger/audit 等数据存在时必须在第一条 DDL 前拒绝，application revision、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0004 -> 0003 -> 0004` 和 check，随后重启 API/Worker。schema downgrade 不是数据平台回迁。 +9. 停止 API/Worker 后在 populated PostgreSQL 上尝试 `20260828_0005 -> 0004`：Worker progress rows 存在时必须在第一条 DDL 前拒绝，application revision、13 表计数、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0005 -> 0004 -> 0005` 和 check，随后重启 API/Worker。历史 `0004` governance/audit guard 继续由 migration 回归覆盖；schema downgrade 不是数据平台回迁。 任何一个场景失败、未运行、使用真实 Provider、最终 PEL/lag 非零或清理不完整，都不能把可靠执行基础写成通过。`--self-check-only` 只验证 Docker/Compose、隔离和清理 guard，不执行九场景，不能替代正式命令。精确 SHA `665244e…` 的最终本地运行已 9/9 通过；artifact 与 hash 见第 10.1 节。 diff --git a/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md new file mode 100644 index 0000000..2be8a0c --- /dev/null +++ b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md @@ -0,0 +1,330 @@ +# ADR-0015：受控可观测性、Worker 进展与审计保留 + +- **Status**: Accepted +- **Date**: 2026-08-28 +- **Deciders**: LLMBenchLab maintainers +- **Scope**: Phase 2 P2-06 metrics exporter、告警、Worker 主循环进展与 audit retention +- **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的任务可观测性、[ADR-0009](ADR-0009-database-governance-audit-fair-scheduling.md) 的 typed audit 保留流程，以及 [ADR-0010](ADR-0010-phase-2-governance-delivery-boundaries.md) 的恢复指标边界 +- **Preserves**: PostgreSQL/数据库事实来源、Redis 非权威通知、`llmbenchlab-protocol-v1`、write-only Provider Key、Mock-only 自动化和可信 loopback 部署边界 + +## Context + +Phase 2 已经交付数据库派生的 `GET /api/v1/tasks/metrics`、在同一数据库读快照内校验 retained audit 的 `GET /api/v1/tasks/history`、稳定分页的 Run audit、Provider/credential 非秘密事件和 Run latency。它们仍有三个正式闭环缺口： + +1. 运维环境没有固定低基数、可被 Prometheus 抓取的 exposition，也没有仓库内可验证的告警规则与响应 Runbook。直接把 rolling-window audit count 标成 counter、把数据库字符串变成 label 或在高频 scrape 中无限扫描 retained event，都会制造错误语义或数据库放大。 +2. `worker_probe.py` 是独立进程执行的数据库/head/Redis capability probe。它不能证明被探测 Worker 的主 event loop 正在 scan、claim、续租或持久化进展；读取任意 peer 的状态再给当前容器报健康也会掩盖卡死。 +3. `audit_events` 已经冻结 `operational >= 90d`、`security >= 365d` 的行内 expiry，但没有可执行的 archive、离线 verify、精确 delete、restore 或 commit-outcome 对账。按 cutoff 宽泛删除、复用业务重放的宽松幂等比较，或把普通文件 hash 宣称为 WORM，都会破坏审计证据。 + +P2-06 只闭合以上应用级可观测性与 audit archive。PostgreSQL 与数据库外 keyring 的配对 backup/restore、PITR/RPO/RTO、Redis 灾难恢复和完整故障矩阵仍属于 P2-07；本决定不能被用来宣称生产 HA、不可篡改存储或 Provider exactly-once。 + +## Decision + +### 1. 事实来源与公共边界 + +- 数据库 UTC、Run/lease/ledger、typed audit 和 Worker progress 行继续是唯一权威事实。Redis ping、Prometheus、告警状态、进程内 single-flight 与日志都不是第二状态机。 +- 新增 `GET /api/v1/metrics/prometheus`。成功响应固定为 Prometheus text exposition `0.0.4`：`text/plain; version=0.0.4; charset=utf-8`、UTF-8、LF、唯一末尾换行和 `Cache-Control: no-store`。 +- Endpoint 不接受查询参数、动态窗口或内容协商；不经 HTTP 递归调用现有 JSON endpoint。现有 `/tasks/metrics` 与 `/tasks/history` 保持兼容并与 exporter 复用同一 snapshot collector。 +- API 与 exporter 仍无认证，只允许可信 loopback/受控内部网络；CORS 不是安全边界。仓库不部署 Prometheus、Alertmanager、OTel collector 或通知发送器。 +- exporter 不输出 Run、Model、Provider、Worker、Question、reservation、policy、request、correlation ID、origin、URL、hash、错误文本或其他用户控制字符串。唯一 labels 是本文列出的固定 enum。 +- 所有 series 都声明为 `gauge`。当前 DB 值和 rolling-window count 都可能下降，不得使用 `_total`、`counter`、`rate()` 或 `increase()` 伪造单调性。 + +### 2. Exporter 快照、窗口和失败语义 + +一次成功 collection 使用一个数据库读快照和一个 DB `current_timestamp`： + +- PostgreSQL：`REPEATABLE READ READ ONLY`；SQLite：首个读查询前显式 `BEGIN`。 +- current Run/governance gauges、expired lease age、audit window、Run latency 与 Worker aggregate 全部使用同一个 DB `now`。 +- audit 窗口固定为最近 `900s` 的半开区间 `[now-900s, now)`，按 `(occurred_at,id)` 最多读取 `50,001` 行；出现第 `50,001` 行时整次 scrape 以 `503 metrics_observation_limit_exceeded` 失败，绝不截断后返回错误 count。 +- Run latency 窗口固定为最近 `3600s`。queue、execution、end-to-end 每类稳定读取最多 `10,001` 个样本，输出前 `10,000` 个的线性 p50/p95/p99 和 `truncated=1`。 +- 每个 API 进程最多一个 collection in flight；重叠请求立即返回 `429 metrics_scrape_in_progress`，不排队。当前单 API 支持拓扑下这是数据库压力保护，不是多进程全局锁。 +- 不提供 last-good cache。数据库或 audit 损坏后返回旧值会掩盖事故。 +- 任一 retained event 的 event contract、payload hash、identity、retention interval 或数值边界损坏时，整次 scrape 返回 `500 audit_event_integrity_error`；数据库异常返回 `503 metrics_database_unavailable`。两者都不返回部分 exposition，也不反射损坏值、DSN 或异常文本。 +- Redis ping 是快照外的非权威 component observation。Redis disabled/down 分别输出 configured/available，DB collection 仍成功；未知 queue 异常同样只把 availability 置零并写固定脱敏日志。 +- 外层 asyncio timeout不能中止已经在线程中运行的同步 DB query；压力边界依赖固定窗口、hard cap、single-flight 和数据库 driver/pool timeout。推荐 scrape interval `>=30s`，本地默认文档使用 `60s`。 + +### 3. 固定 metric 集 + +当前 DB gauges 不带 label： + +```text +llmbenchlab_runs_pending +llmbenchlab_runs_due_pending +llmbenchlab_runs_running +llmbenchlab_runs_expired_running +llmbenchlab_runs_cancellation_requested +llmbenchlab_runs_retry_scheduled +llmbenchlab_runs_dead_lettered +llmbenchlab_runs_queue_notification_error +llmbenchlab_runs_managed_backlog +llmbenchlab_runs_governance_delayed +llmbenchlab_runs_governance_exhausted +llmbenchlab_provider_attempts_active +llmbenchlab_governance_scopes_overdrawn +llmbenchlab_run_lease_acquisitions +llmbenchlab_run_failed_attempts +llmbenchlab_run_dispatches +llmbenchlab_run_expired_lease_oldest_age_seconds +``` + +expired lease age 是 `max(0, DB now - min(expired running lease_expires_at))`；空集合为零。它只表示当前 right-censored 逾期年龄，不是历史 lease-recovery duration 分布，也不改变 ADR-0010 的暂缓边界。 + +Typed audit 固定输出全部 17 个 event type，包括零值： + +```text +llmbenchlab_audit_events_window{event_type="..."} +llmbenchlab_audit_event_window_seconds +llmbenchlab_metrics_audit_events_scanned +llmbenchlab_metrics_audit_event_limit +``` + +固定 event type 为 `governance_policy_bootstrapped`、`governance_policy_applied`、`run_admitted`、`run_claimed`、`run_cancel_requested`、`run_deferred`、`run_yielded`、`run_terminal`、`run_retry_scheduled`、`run_dead_lettered`、`run_lease_reconciled`、`provider_attempt_reserved`、`provider_attempt_send_started`、`provider_attempt_settled`、`question_evidence_persisted`、`queue_notification`、`governance_integrity_error`。 + +Run latency 固定为： + +```text +llmbenchlab_run_latency_quantile_seconds{phase="queue|execution|end_to_end",quantile="0.5|0.95|0.99"} +llmbenchlab_run_latency_samples{phase="queue|execution|end_to_end"} +llmbenchlab_run_latency_truncated{phase="queue|execution|end_to_end"} +llmbenchlab_run_latency_window_seconds +llmbenchlab_metrics_latency_sample_limit +``` + +无样本时输出 `samples=0`、`truncated=0`，不输出 quantile series；不得以零、NaN 或 Infinity 冒充观测。 + +Queue 与 Worker 固定为： + +```text +llmbenchlab_queue_configured +llmbenchlab_queue_available +llmbenchlab_worker_processes{state="registered|live|stalled"} +llmbenchlab_worker_expected_minimum +llmbenchlab_worker_shortfall +llmbenchlab_worker_activity_observed{activity="scan|claim|lease_heartbeat|progress"} +llmbenchlab_worker_activity_oldest_age_seconds{activity="scan|claim|lease_heartbeat|progress"} +llmbenchlab_worker_stale_threshold_seconds +llmbenchlab_run_recovery_alert_threshold_seconds +llmbenchlab_metrics_snapshot_unixtime_seconds +``` + +没有某类 Worker event fact 时仍输出 observed=0，但省略对应 age series。Renderer 只接受有限非负数字并按固定顺序生成 HELP/TYPE/sample；任何负值或非有限值使 collection fail closed。 + +JSON `/tasks/metrics` 的五个时间字段按未停止 generation 取 `MAX`，表示最近一次聚合事实；Prometheus 的 `worker_activity_oldest_age_seconds` 则对四类 activity 先取未停止 generation 的 `MIN` 再以同一 DB now 计算年龄，表示最老 active-generation 事实。两者语义不得混用或用一个聚合值冒充另一个。 + +### 4. Worker process/progress 数据模型 + +新增 `worker_processes` 表： + +| 字段 | 合同 | +| --- | --- | +| `generation_id VARCHAR(36) PK` | 每次长运行 Worker 启动生成的 canonical UUID，永不复用 | +| `worker_id VARCHAR(128) NOT NULL UNIQUE` | 现有 lease/audit owner，仅内部关联，不从 API/exporter 输出 | +| `started_at UTC NOT NULL` | 注册事务的 DB UTC | +| `last_seen_at UTC NOT NULL` | 最近成功刷新的任一真实 event | +| `last_scan_at UTC NULL` | reaper 与 due-run scan 均成功 | +| `last_claim_at UTC NULL` | lease claim commit/后置完整性处理成功 | +| `last_progress_at UTC NULL` | Response 新插入或 Run/reaper durable 状态实际推进 | +| `last_lease_heartbeat_at UTC NULL` | 已有 lease 成功续租；初始 claim 不计 | +| `stopped_at UTC NULL` | graceful stop 的 DB UTC | + +约束：worker_id 长度 `1..128`；`last_seen_at >= started_at`；四个可空 event time 均在 `[started_at,last_seen_at]`；`stopped_at IS NULL OR stopped_at >= last_seen_at`。对 stopped generation 的任何迟到 flush 都由 `generation_id AND stopped_at IS NULL` CAS 拒绝。表不保存 hostname、PID、current Run、Provider、URL、错误文本、题目或 Response 内容。唯一辅助索引为 `(stopped_at,last_seen_at,generation_id)`。 + +生命周期只有 `registered(active) -> stopped`。`stale` 是同一 DB snapshot 中的派生分类，不写成可被迟到 event 复活的状态；异常退出保留 `stopped_at=NULL` 并自然变 stale。 + +配置冻结为： + +```text +worker_progress_flush_seconds = 5 +worker_progress_stale_seconds = 60 +worker_expected_processes = 1 +worker_recovery_alert_seconds = 60 +``` + +`stale_seconds` 必须至少为 `3 * max(flush_seconds, worker_poll_seconds, redis_block_milliseconds/1000, worker_heartbeat_seconds)`。Compose 默认一个 Worker，扩到两个时操作者必须同时把 expected minimum 设为 2；只有明确不需要执行能力的环境才可设零，不能从历史行数猜测 expected count。 + +### 5. Worker recorder 与 event 定义 + +长运行 `WorkerService.run()` 启动前必须完成 generation 注册；注册失败拒绝启动不可观测 Worker。`run_once()`、probe、测试/CLI 工具不注册长运行 generation。 + +每个 Worker 使用一个 event-loop owned、thread-safe note、单 in-flight 的 coalescing recorder： + +- scan/claim/progress/lease-heartbeat 只设置固定 bit；后台最多每 5 秒通过 `asyncio.to_thread` 执行一笔短事务。 +- 一次 flush 读取一个 DB `current_timestamp`，将全部 pending event 字段和 `last_seen_at` 写为同一值。timer 本身绝不能刷新 seen；无真实 bit 时零写入，避免主循环已死但 keepalive 继续制造幽灵健康。 +- commit 成功后才清 bit；失败保留并重试，只记录固定错误。运行期 progress 失败不回滚 Run、Response 或 lease，旧时间会保守变 stale。 +- graceful stop 将最后 pending bit 与 `stopped_at` 在同一短事务写入；失败保守留下 stale generation。 +- observer 是可空、no-throw 的固定 enum 接口；既有 Runner/Worker 测试和调用方缺省使用 null object。 + +event 定义： + +- `scan`：`_reap_expired()` 与 `_due_run_ids()` 都成功；任一步 DB 失败不记录。 +- `claim`：Runner 得到非空 lease 且 claim commit 与 post-commit integrity 处理成功；no-op 不记录。 +- `lease_heartbeat`：已有 lease 成功续租；rejected/exception/初始 claim 不记录。 +- `progress`：Response `INSERTED`；terminal/defer/exhaust/yield/retry/dead-letter/cancel transition commit 确认；或 reaper report 任一 durable 计数非零。`ALREADY_PRESENT`、scan、heartbeat、Redis delivery/ACK 不算 progress。 + +Worker probe 保持 capability-only，不读取 peer aggregate、不写 progress，也不把任意 generation 冒充当前容器。响应固定补充： + +```json +{"probe_scope":"dependencies_only","main_loop_progress":"not_checked"} +``` + +没有由主进程交付的 exact generation handoff 前，Docker healthcheck 不得声称当前 process main-loop liveness。 + +### 6. Worker read model 与 stale 语义 + +在 `/tasks/metrics` 与 exporter 的同一 DB `now` 中： + +```text +cutoff = now - worker_progress_stale_seconds +registered = stopped_at IS NULL +live = registered AND last_seen_at >= cutoff +stalled = registered AND last_seen_at < cutoff +shortfall = max(worker_expected_processes - live, 0) +``` + +边界 `last_seen_at == cutoff` 为 live。JSON task metrics 增加 expected/registered/live/stalled/shortfall/stale-after，以及未停止 generation 中五个 event timestamp 的 `MAX`；空 fact 为 `null`。响应绝不包含 generation/worker ID。 + +历史 stopped 与 stale process 行属于低量运维事实，不是 audit/WORM。本切片不在 scrape/API 请求链路删除它们；importer 只拒绝仍 live 的 generation，并精确复制 stopped/stale 行。旧 crash 行即使存在，也不能让 `shortfall` 消失。需要 downgrade 时先停止 Worker、保存需要的运维事实并显式清空表；不得由 migration 静默丢弃。 + +### 7. 告警规则与 Runbook + +新增 `deploy/observability/prometheus-alerts.json`；JSON 是 YAML 子集，可由 Prometheus rule loader 使用，也可由 Python 标准库严格测试，不新增 PyYAML。新增仅作示例的 scrape 配置，固定 `job_name: llmbenchlab`、path `/api/v1/metrics/prometheus`、interval `>=30s`；不把 Prometheus/Alertmanager 加入 Compose。 + +规则组 `llmbenchlab.phase2`、interval `30s`，固定八条： + +| Alert | expression | for | severity | Runbook anchor | +| --- | --- | ---: | --- | --- | +| `LLMBenchLabMetricsUnavailable` | `up{job="llmbenchlab"} == 0` | `2m` | critical | `alert-exporter-unavailable` | +| `LLMBenchLabBacklogPersistent` | `llmbenchlab_runs_due_pending > 0` | `15m` | warning | `alert-backlog-persistent` | +| `LLMBenchLabDeadLettered` | `llmbenchlab_audit_events_window{event_type="run_dead_lettered"} > 0` | `0s` | critical | `alert-dead-letter` | +| `LLMBenchLabGovernanceIntegrityError` | `llmbenchlab_audit_events_window{event_type="governance_integrity_error"} > 0` | `0s` | critical | `alert-governance-integrity` | +| `LLMBenchLabGovernanceOverdrawn` | `llmbenchlab_governance_scopes_overdrawn > 0` | `0s` | critical | `alert-governance-overdraw` | +| `LLMBenchLabQueueDegraded` | `llmbenchlab_queue_configured == 1 and llmbenchlab_queue_available == 0` | `2m` | warning | `alert-queue-degraded` | +| `LLMBenchLabWorkerStalled` | `llmbenchlab_worker_shortfall > 0` | `2m` | critical | `alert-worker-stalled` | +| `LLMBenchLabLeaseRecoverySlow` | `llmbenchlab_run_expired_lease_oldest_age_seconds > llmbenchlab_run_recovery_alert_threshold_seconds` | `1m` | warning | `alert-lease-recovery-slow` | + +每条规则只有 `severity` 与固定 `component=llmbenchlab-control-plane` label，并具有 summary、description、绝对 runbook URL 和 silence_policy annotation。Runbook 使用显式英文 HTML anchor，逐条规定响应、证据、恢复和以下 silence 边界：只允许有 owner/ticket/到期时间的窄 matcher；governance integrity/overdraw 不作常规静默；维护静默最长 15 分钟至 4 小时（按事件表固定），禁止全局 silence。 + +Dead-letter 与 integrity 是 15 分钟 rolling symptom；Prometheus 整个窗口不可用时可能错过一次性事件。Run audit 仍是事实来源，规则不等于 durable acknowledgement。 + +### 8. Audit 完整行验证 + +把 `runs.py` 与 `health.py` 重复的 retained-row read validation 收敛到 `app.governance.audit` 的公共函数。API、history、exporter、archive 和 restore 都必须复用同一 event contract、payload hash、identity、retention interval、timestamp 和 numeric validation。Read validation 不只比较 normalize 后的值：数据库原始 JSON 必须与 normalize 后的 canonical JSON（类型和编码语义）完全一致；例如数值 `1` 不得冒充规范化的 USD 字符串 `"1"`，否则 archive 会改写而不是保存原始存储事实。 + +业务 `append_audit_event()` 的 `_event_matches()` 有意忽略首次观察的 `id/occurred_at/expires_at`，适合 commit-ack 重放；archive restore 的精确比较必须包含所有存储字段，绝不能复用该宽松函数。 + +### 9. Audit archive 与文件合同 + +`llmbenchlab-audit-retention archive --output PATH` 在一个数据库读快照中取得 DB UTC `cutoff_at`，只选择 `expires_at < cutoff_at`： + +- PostgreSQL 使用 `REPEATABLE READ READ ONLY`；SQLite 首读前显式 `BEGIN`。 +- 按 `(expires_at,id)` 排序，单批固定上限 `10,000`，额外一行只决定 `has_more_eligible`。 +- cutoff 不由用户任意传入；空集合也生成有效 archive。 +- archive 默认绝不删除；运维顺序固定为 archive -> offline verify -> maintenance-window delete。 + +单文件 canonical JSONL schema 为 `llmbenchlab-audit-archive-v1`：一行 header、零至一万行 `audit_event`、唯一末行 manifest。V1 的 event/payload/retention contract 与 compatible Alembic head allowlist 独立冻结；当前唯一 compatible head 是 `20260828_0005`。`source_alembic_head` 不是装饰性字符串：write/verify/restore/delete/reconcile 都拒绝未列入 allowlist 的旧版、分支或未来 head；未来 schema 只有在证明 V1 全字段语义仍兼容后才能显式扩充 allowlist，否则必须提升 archive schema 并保留 V1 reader。Event 保存完整恢复事实： + +```text +id, event_key, event_type, payload_hash, payload, retention_class, +occurred_at, expires_at, correlation_id, run_id, model_id, question_id, +worker_id, reservation_id, attempt, provider_attempt, lease_token, +duration_ms_hex +``` + +时间编码为六位微秒 UTC `Z`；finite nonnegative duration 用规范化 `float.hex()`，正负零统一。Manifest 至少含 event/type/class counts、occurred/expiry min/max、`has_more_eligible`、source Alembic head 和 `content_sha256`。Content hash 对 header+event canonical bytes 使用域分隔和长度前缀；manifest 不进入自引用 hash。工具另计算整个文件 `archive_sha256`，delete/restore 必须提交该精确 digest。 + +这两个 hash 只提供误改检测与精确文件绑定，不是签名、认证或 WORM。 + +Canonical 与文件边界： + +- UTF-8、LF、最终换行；JSON `sort_keys=True`、紧凑分隔符、`ensure_ascii=True`、`allow_nan=False`；每行必须与重新编码 bytes 完全相同。 +- 文件最大 128 MiB、单行最大 64 KiB、事件最多 10,000。拒绝 duplicate JSON key、空行、尾随记录、未知/缺失字段、NaN/Infinity、非法 UTF-8、重复 id/event_key、乱序和 hash/rollup 漂移。 +- 输入必须是当前用户拥有、权限不宽于 `0600` 的普通非 symlink 文件。输出父目录必须存在；目标不得存在。 +- 同目录用 `O_EXCL|O_NOFOLLOW` 创建 `0600` 临时文件，write+fsync 后 no-replace 原子安装，再 fsync 父目录。失败不留下可被误认作成功的目标。 +- verify/delete/restore 一次打开并解析同一个 file descriptor，避免验证后换文件。 + +Archive 含内部 Run/Model/Question/Worker 等运维 ID，按敏感运维文件保护，不经 API 提供下载。它不得包含 Key、Authorization、Cookie、credentialed DSN、credential ciphertext/nonce、keyring、Provider URL、题目正文、Prompt、Response 或 raw Provider body。 + +### 10. Verify、reconcile、restore 与 delete + +CLI 固定为： + +```text +llmbenchlab-audit-retention archive --output PATH +llmbenchlab-audit-retention verify --archive PATH [--expected-sha256 HASH] +llmbenchlab-audit-retention reconcile --archive PATH --confirm-sha256 HASH +llmbenchlab-audit-retention restore --archive PATH --confirm-sha256 HASH +llmbenchlab-audit-retention delete --archive PATH --confirm-sha256 HASH +``` + +数据库连接只从项目固定 settings/environment 取得；命令不接受 DSN 字符串参数。`verify` 完全不导入数据库 session/engine、不读取或校验数据库配置、不创建 SQLite 父目录，也不连接数据库；即使数据库 URL 无效或不可用，合法 archive 仍可在 fresh process 中验证。它只输出固定 status、count 和 digest，不输出 path、ID、payload 或异常原文。参数解析错误同样使用固定 exit `2`/code；未知参数和值不得由 argparse usage/error 反射。 + +Restore/delete/reconcile 先在同一 file descriptor 完成 strict verify 与 digest confirmation，再打开当前 Alembic head 数据库。PostgreSQL mutation 使用固定 transaction advisory lock，SQLite 使用 `BEGIN IMMEDIATE`，以串行化本工具的 mutation；API/Worker audit writers仍必须在 delete/restore 维护窗口停止。 + +Restore 对每条 record 同时按 id 与 event_key 比较全部存储事实： + +- 二者均不存在：插入精确原行； +- 已存在且全部字段 exact：幂等 no-op； +- 同 key 不同 id、同 id 不同 key、任一字段不同或关联 reservation 缺失：整批 rollback； +- exact 与 absent 可以在同一事务安全收敛；提交后独立只读快照必须全部 exact。 + +Delete 永不执行宽泛 `DELETE WHERE expires_at < cutoff`： + +- DB now 不得早于 archive cutoff，且每条 archive record 必须已经过期； +- 事务内 exact id/key 锁定并比较；全部 exact 时只删除 archive 列出的行；全部 absent 返回 `already_absent`；mixed missing/conflict 整批 rollback； +- 事务内删除数必须等于 archive count，提交后独立快照必须全部 absent；不删除 ledger、Run、Response 或其他表。 + +`reconcile` 是只读恢复入口，只返回 `all_exact`、`all_absent`、`mixed_exact_absent`、`conflict` 或 `empty_archive`，不打印 event 正文。 + +提交结果沿用 importer 风格：exit `0` 为成功且后验验证完成；`2` 为 pre-commit/file/preflight 失败且无已知写入；flush 成功后 commit 抛错为 `4 commit_outcome_unknown`，禁止盲重试；commit 返回后后验验证失败为 `3 committed_but_verification_failed`。所有错误只输出固定 code、operation、count/digest 和异常类型，不使用可能含 path/DSN/payload 的 `str(exc)`。 + +### 11. Schema、migration 与 importer + +新 Alembic head `20260828_0005`，同时是 archive-v1 当前唯一 compatible source/target head： + +- 创建 `worker_processes` 及本文约束/索引； +- 为 audit archive 扫描增加 `(expires_at,id)` 索引； +- 为 exporter 的有界 audit 窗口扫描增加 `(occurred_at,id)` 索引； +- 不创建 retention 状态表，不回填旧 Worker 进展，也不修改历史 audit expiry。 + +`prepare_migrations.py` 将 `20260827_0004` 纳入 historical head/fingerprint/备份/adoption，严格验证新表、约束和索引。`0005 -> 0004` 在第一条 DDL 前要求 `worker_processes` 为空；有行即拒绝且不得静默丢失事实。空隔离库继续支持 downgrade/re-upgrade。 + +SQLite -> PostgreSQL importer 的 core table list 从 12 增至 13，精确复制 stopped 和已经 stale 的 Worker facts并纳入 count/PK/content digest、目标空表与锁序；source preflight 使用 DB now/stale threshold拒绝仍 live 的 generation，防止把正在运行的 Worker 数据库迁走。Stale crash row不会被当成 live，也不会让 expected shortfall 消失。 + +Audit archive 文件不是 importer 的第十四张表，也不由 importer 自动读取。P2-07 后续整库恢复必须独立证明 13 表与 keyring 配对；本决定不提前完成该工作。 + +### 12. 日志与秘密边界 + +- 新 exporter、Worker recorder、archive 和 CLI 只记录固定 event/error code、operation、计数和 allowlisted enum；不得记录 DSN、path、event payload、record ID、worker ID、hash 以外的文件内容或原始异常文本。 +- 维护 CLI 的 digest 可以输出；archive path、内部 ID 和 child/raw 诊断不进入公开状态文档。 +- 所有生产 logging source 必须在本切片中做静态和动态 marker 审查；只有显式登记的应用 logger 可输出 literal message，structured extra 还须逐字段经过固定 enum、canonical UUID/Redis stream ID、HTTP method/route 与有限数值规范化。非法 ID 省略，未知字符串只输出固定 `unsupported`；Redis notification 的 Run/correlation identity 在进入 Worker 前也必须是 canonical UUID。上游异常、OS error、SQLAlchemy error 和 subprocess output 不直接进入 message/extra。 +- 自动化只使用 Mock/Stub，不创建真实 Provider Model，不继承或调用真实 Provider credential。 + +## Consequences + +### Positive + +- JSON API、Prometheus scrape、告警与 Worker liveness 从同一 DB-time facts 派生，且低基数/压力/损坏边界可测试。 +- capability probe 与 main-loop progress 不再混淆；长 SSE 可用 lease heartbeat 保持 live，而无事件 timer不能制造幽灵健康。 +- audit 保留从“只有 expiry 字段”变成 archive、离线验证、精确删除、恢复和不确定提交对账的可执行流程。 +- 无新增 Prometheus/YAML/OTel 生产依赖，数据库仍是唯一事实来源。 + +### Negative + +- 每个长运行 Worker 最多每 5 秒增加一笔短事务；高频 scrape 仍会读取有界 audit/Run 窗口，需要按文档限制抓取频率。 +- Stale crash generation 会保留为诊断事实；数据库迁移与 downgrade 需要显式处置，而不是静默删除。 +- JSONL hash 不能抵抗有权同时改文件与 hash 的管理员；Alert rules 没有 sender/ack 状态，Prometheus 长时间不可用时可能错过 rolling event。 +- Audit archive restore 只恢复 audit 行，不证明数据库、credential keyring、Redis 或 Provider 侧状态完整。 + +## Validation + +- Worker：注册/停止、DB UTC、bit coalescing、无 event 零写、flush failure重试、stopped CAS、scan/claim/heartbeat/progress精确边界、stale/expected/shortfall、probe capability-only，SQLite/真实 PostgreSQL并发一致。 +- Exporter：固定 content type/顺序/HELP/TYPE、空与 populated 快照、17 event series、窗口边界、50,001 hard cap、latency cap/分位数、queue disabled/down、Worker stale、长 SSE、single-flight、audit/DB/renderer fail closed、marker/ID/secret absence。 +- Rules：标准库 strict JSON、精确 8 rules、只引用已交付 metrics/`up`、固定 labels/annotations/runbook anchors、event rules禁用 rate/increase；若本机有 promtool，额外运行 `promtool check rules` 并如实记录。 +- Retention：canonical/hash/rollup/permissions/symlink/atomic output/limits、冻结 V1 fixture/contract 与 unsupported future head 拒绝、fresh-process invalid DB URL 下离线 verify/无 engine 或目录副作用、argparse marker 不反射；SQLite archive->verify->delete->reconcile->restore；same-key/different-fact、same-id/different-key、missing reservation、mixed batch、commit unknown；真实 PostgreSQL snapshot/advisory/row-lock/完整往返。 +- Migration/importer：0004 adoption/backup到0005、双方言空库 round-trip、populated downgrade拒绝、13表 exact digest、live Worker preflight拒绝。 +- 完成目标/全量 pytest、Ruff、Mock smoke、Compose config、文档链接、staged diff/secret检查后，形成独立 commit push，并等待该精确 SHA 的四个必需 CI job全绿。P2-07未完成时 Phase 2保持 `in_progress`。 + +## Rollback + +先停止 API/Worker/exporter scrape 和 retention maintenance。应用可关闭 exporter 和 Worker progress recorder，但保留 `0005` schema、已有 progress rows和 archive；不允许为代码回滚删除审计或未保存的 Worker facts。只有 `worker_processes` 为空时才能 downgrade 到 `0004`。 + +Audit delete默认从不自动运行。若 delete/restore出现 exit 3/4，禁止盲重试，先用同一 archive+digest执行只读 reconcile。已经生成的 archive按敏感 `0600` 文件保存并独立验证；它不能替代 PostgreSQL+keyring备份。若 exporter压力或损坏导致非2xx，修复数据库/降低抓取频率，而不是返回缓存或放宽完整性检查。 diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 45755e8..9b621bf 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -12,6 +12,7 @@ - 单机资格：[ADR-0012](../decisions/ADR-0012-single-host-slo-capacity-qualification.md) - 镜像指纹修正：[ADR-0013](../decisions/ADR-0013-stable-image-content-fingerprint.md) - 双 backlog 资格：[ADR-0014](../decisions/ADR-0014-dual-backlog-slo-profile.md) +- 可观测性与审计保留：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) ## 阶段目标 @@ -19,14 +20,15 @@ ## 当前功能范围 -- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容、双方言 Alembic 链至 `20260827_0004`。 +- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容、双方言 Alembic 链在当前 P2-06 工作树扩展至 `20260828_0005`。 - Redis at-least-once 通知；Run、取消、重试、租约、Response、终态、治理、attempt ledger 和 audit 全由数据库裁决。 - 原子 claim、数据库时间 lease/heartbeat、fencing、有限 retry/backoff、取消、过期接管、duplicate no-op 和 dead-letter。 -- 停写只读 SQLite→空 PostgreSQL 的单向 importer；`0004` 后按依赖顺序复制 12 张应用表，并做 count/PK/content fingerprint。keyring 仍在数据库之外。 +- 停写只读 SQLite→空 PostgreSQL 的单向 importer；`0005` 按依赖顺序复制 13 张应用表并做 count/PK/content fingerprint，源有 live Worker generation 时拒绝，stopped/stale progress 可精确复制。keyring 仍在数据库之外。 - managed API Run 的 active policy ID/hash 与显式 input/Token/cost override 冻结；global/provider/model/run 四层 concurrency、固定分钟 RPM/TPM、global/run lifetime request/Token/USD budget。 - 每个 Provider HTTP attempt 的 reserve→send-started→actual/conservative settlement 或 confirmed pre-send release never-delete ledger；materialized scope/bucket counter 仅作投影，任何高/低漂移 fail closed。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model due ordering。 - typed audit、分页 Run audit、task history counters、基于 Run 数据库时间戳的 queue/execution/end-to-end latency、严格规范化 Provider metadata 和非秘密 credential audit。 +- 固定低基数 Prometheus text exporter、八条仓库内告警规则/Runbook、DB-time Worker generation/progress 聚合、canonical audit archive/离线 verify/reconcile/restore/delete 和全日志源治理；这些 P2-06 功能已实现，仓库级全门禁与远程门禁仍待收尾。 - Run Detail 展示 managed/delayed/exhausted、治理原因和明确 UTC not-before；旧 Run 与可信本地 CLI 明确为 `legacy_unmanaged`。 - 真实 PostgreSQL 竞争测试及 Mock-only enhanced capacity/acceptance；精确实现 SHA 的完整 capacity、9/9 acceptance、`P2-local-control-plane-v2` 多轮单机资格与远程 4/4 CI 已通过。 @@ -35,7 +37,7 @@ - 不新增标准 Benchmark、代码沙箱、Judge、Arena、Agent、认证、多租户或公共部署。 - 不把本地 ledger/Response 幂等描述为 Provider 请求或账单 exactly-once。 - 不把 fixed-minute limiter 描述为平滑 token bucket，不把 Mock 容量描述为真实 Provider 或生产 SLA。 -- 单机资格不交付生产 SLA、Prometheus/OTel Exporter、告警规则、audit retention archive、备份恢复认证或 Worker 主循环进展探针。 +- 单机资格不交付生产 SLA；P2-06 也不部署 Prometheus/Alertmanager/OTel/通知发送器，不提供 WORM、自动 audit 删除、整库/keyring 备份恢复认证或当前容器 exact-generation health handoff。 - 不改变题目、评分分母、聚合、排行榜隔离或 protocol-v1 语义。 ## 支持边界与不变量 @@ -59,11 +61,11 @@ | ID | 状态 | 已交付与剩余范围 | | --- | --- | --- | | P2-01 一致性与容量设计 | `completed` | ADR-0012～0014、DB truth/lease/fencing/治理、v2 四 cell 多轮统计、恢复与连接模型已交付；clean SHA `b6a35fe…` 的 1+5 资格为 23/23、`qualified`；证据文档 commit `875f13a…` 已 push，精确 SHA CI 4/4 成功 | -| P2-02 PostgreSQL 迁移 | `slice_delivered` | `0002`/`0003` 可靠性与凭据基础、`0004` 治理/审计 schema、12 表 importer 与双方言测试已实现；本地及精确 SHA 远程真实 PG migration/check/integration 已通过，无自动反向回迁 | -| P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；Worker 主循环 progress/liveness 事实仍未交付 | +| P2-02 PostgreSQL 迁移 | `slice_delivered` | 历史 `0002`～`0004` 与 12 表 importer 已通过精确 SHA 远程门禁；当前工作树增加 `0005` / 13 表 importer、live Worker preflight 和 populated downgrade guard，待 P2-06 最终门禁 | +| P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；当前工作树实现 generation 级 DB-time scan/claim/lease-heartbeat/progress 与 stale 聚合，dependency probe 仍只表示 capability | | P2-04 生命周期可靠性 | `foundation_delivered` | retry/backoff、取消、恢复、dead-letter、Response 幂等和三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；Provider 外部副作用仍为 at-least-once | | P2-05 并发治理 | `slice_delivered` | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序、counter 重算 fail-closed 与 ADR-0011 已实现；精确 SHA 的真实 PG/capacity/acceptance/CI 候选门禁已通过 | -| P2-06 可观测性 | `slice_delivered` | DB gauges、typed audit/history、Run latency、Provider metadata、credential audit 和 UI 状态已实现；Exporter/告警、retention archive、Worker progress/liveness、全日志源治理仍未完成 | +| P2-06 可观测性 | `in_progress` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已实现；lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已通过，commit/clean-SHA Compose/push 与精确 SHA CI 仍未完成 | | P2-07 验证与运维 | `slice_delivered` | enhanced capacity/PG tests、正式 v2 单机资格、Operations/Performance/Deployment 与精确 evidence 已交付；backup/restore、完整失败矩阵与告警响应仍未完成 | `slice_delivered` 表示该垂直切片及其候选门禁已交付，不表示整个阶段完成。Phase 2 必须保持 `in_progress`。 @@ -78,7 +80,7 @@ - [x] **crash seam 验收**：`reserved`→send-start、`send_started`→settlement、Response commit→最终恢复三条 deterministic DB seam injection 在完整 Compose acceptance 通过；它们不冒充精确时刻 `SIGKILL`。 - [x] **远程实现门禁**：GitHub Actions run [`33099260233`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33099260233) 对精确实现 SHA 4/4 成功。 - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 -- [ ] **P2-06 正式闭环未通过**：没有 exporter/告警、audit retention archive 或 Worker progress/liveness；现有 dependency probe 不能证明主循环正在推进。 +- [ ] **P2-06 仓库级闭环未通过**：范围内功能与 lint/test/smoke、真实 PostgreSQL/Redis、双方言 migration、frontend build、Compose config、八规则 promtool、dirty capacity/9/9 acceptance 和修复后 staged 技术/安全终审已通过；仍须形成独立 commit、在 clean SHA 重跑 Compose、push 并等待该精确 SHA CI 全绿。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -97,16 +99,24 @@ | 正式 v2 单机资格 | `b6a35fe…` 上 1+5、23/23；aggregate SHA-256 `a76d167b…d0d9` | 每轮 22/330/330/331、hard invariant 与 exact-project cleanup 通过；Mock-only 非生产 SLA | | 远程 CI | run `33146681285` 4/4 | 精确 v2 实现 SHA 全绿；PR #2 未合并 | | 设计/计时修复远程 CI | SHA `1cd19c51ed309316047a18ed3b2a308647af495d`，run `33081854406`，4/4 | 不包含当前治理实现 | +| P2-06 lint/test | `make lint` 全绿（Ruff 152 files、ESLint、TS）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed` | 当前未提交工作树通过；只用 Mock/Stub | +| P2-06 smoke/integration/migration/build/config | smoke `1 passed, 7 deselected`；临时 PG16/Redis7 migration/check 后 integration `33 passed, 0 skipped`；临时 SQLite head→`0001`→head/check、frontend build（2192 modules，保留 662.39 kB warning）、Compose config 全绿 | 默认用户 SQLite 未到 head且未擅自迁移；首次 integration cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；当前结果尚未绑定远程 SHA | +| P2-06 规则门禁 | 临时 `prom/prometheus:v3.5.0` 容器中 `promtool check rules` 成功 | 八条规则全部通过；不表示仓库部署 Prometheus/Alertmanager | +| P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`；`0005` populated 与 isolated `0004` refusal、两层空库往返、cleanup C/V/N empty 均通过 | +| P2-06 dirty capacity | 最新 artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453` 通过 | 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit；0 error/drift/duplicate/PEL/lag，expected Worker 2、cleanup C/V/N/image 0；offline Mock、非 SLO | +| P2-06 补充 Ruff | 过宽 scripts 命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | 保留首次结果，不扩大本切片清理范围 | +| P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 当前 dirty 工作树终审通过 | +| P2-06 剩余门禁 | commit、clean-SHA Compose、push/精确 SHA CI 待完成 | 不得借用上述历史绿色结果；保持 `in_progress` | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 ## 迁移与回滚边界 -- importer 的 12 表依赖顺序包含原六张业务/凭据表和六张 governance/audit 表；源必须停写且可读，目标必须空，copy/对账在单一目标事务内完成。 +- importer 的 13 表依赖顺序包含原六张业务/凭据表、六张 governance/audit 表和 `worker_processes`；源必须停写且可读、不得有 live Worker generation，目标必须空，copy/对账在单一目标事务内完成。 - active policy/reservation、materialized/ledger 漂移和 schema fingerprint 不满足 preflight 时 fail closed。summary 只含 row count、PK/content digest，不输出业务内容或凭据材料。 - keyring 不随数据库复制；恢复目标必须另行安全取得匹配 keyring。数据库与 keyring 同时泄漏可恢复 Provider Key，丢失 keyring 则 stored credential 不可恢复。 - exit `2` 为提交前回滚；exit `4` 为 COMMIT 结果未知；exit `3` 为提交已确认但后验证/输出失败。exit 3/4 禁止盲目重试。 -- downgrade `0004` 前必须停止 API/Worker、关闭 admission、对账 active reservation，并确保六类新表和 managed active Run 满足 guard；不得为代码回滚删除已结算 ledger/audit。 +- downgrade `0005` 前必须停止 Worker、保存所需 progress 并显式清空 `worker_processes`；migration 不会静默丢弃 process facts。进入 `0004` 后，仍必须停止 API/Worker、关闭 admission、对账 active reservation 并满足原六类 governance/audit guard；不得为代码回滚删除已结算 ledger/audit。 ## 风险与剩余控制 @@ -115,15 +125,15 @@ | 限额并发突破 | canonical scope、固定锁序、DB transaction、ledger 重算、真实 PG integration 与 v2 多轮资格 | 超出固定单机 profile 时重新测量并持续回归 | | Provider 调用/费用重复 | send-start marker、保守结算、本地幂等及三条 crash seam acceptance | 外部 exactly-once 不可承诺 | | 长 Run 饥饿 | finite quantum、due ordering、dispatch/failure 分离及 v2 每轮公平性硬门禁 | 更大规模或不同 Worker 拓扑需重新建模 | -| Worker 停滞不可见 | dependency probe、typed current gauges | DB-time progress/liveness、exporter 与 alert | -| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination | retention archive/restore 和 cardinality 运行边界 | +| Worker 停滞不可见 | DB-time progress/liveness 聚合、exporter 与 `WorkerStalled` rule；dependency probe 保持 capability-only | 完成 P2-06 门禁，并在 P2-07 演练告警处置/扩缩 | +| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination、canonical archive/verify/精确 delete/restore | 完成 P2-06 门禁；P2-07 验证异地存储与整库恢复边界 | | 容量结论过度外推 | Mock-only、环境/config/evidence 指纹、1+5 多轮和明确支持 profile | 不冒充 Provider/生产 SLA；环境或 profile 变化必须重新资格 | -| 灾难恢复失败 | 12 表 importer、迁移 guard、独立 keyring 边界 | PostgreSQL backup/restore、keyring 配对和 audit archive 演练 | +| 灾难恢复失败 | 13 表 importer、迁移 guard、audit archive 自身 restore、独立 keyring 边界 | PostgreSQL backup/restore、keyring 配对、Redis 重建与完整恢复演练 | ## 交付物与下一任务 -已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 继续 Exporter/告警、retention archive、Worker progress/liveness 和 backup/restore。 +已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。当前 P2-06 工作树另已实现 `0005` / 13 表 importer、Worker DB-time progress、exporter/八规则、audit retention CLI 与日志治理，dirty capacity/9/9 acceptance 和最终 review 已通过；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 先完成 commit/clean-SHA Compose/精确 SHA CI，再推进 P2-07 backup/restore 和剩余运维演练。 ## 状态 -`in_progress`。P2-01 已完成，P2-05 已交付，P2-06/P2-07 也已有治理/审计/容量切片和远程门禁，但 Exporter/告警/retention/backup/Worker-progress 等阶段验收仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、完整可观测性、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +`in_progress`。P2-01 已完成，P2-05 已交付；P2-06 功能已实现但仓库级门禁未闭环，P2-07 backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 diff --git a/docs/plans/2026-08-28-phase-2-observability-retention.md b/docs/plans/2026-08-28-phase-2-observability-retention.md new file mode 100644 index 0000000..51e0cf8 --- /dev/null +++ b/docs/plans/2026-08-28-phase-2-observability-retention.md @@ -0,0 +1,141 @@ +# Phase 2 可观测性与审计保留执行计划 + +- Owner: Codex +- Status: active +- Created: 2026-08-28 +- Updated: 2026-08-28 +- Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- Worklog: [2026-08-28 工作日志](../worklogs/2026-08-28-phase-2-observability-retention.md) +- ADRs: [ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](../decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) + +## Context + +Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audit/history 和单机 Mock 控制面资格。本计划开始时，数据库 API 能返回任务 gauges、typed event counters 和 Run latency，但没有稳定低基数 exporter/规则、Worker 主循环进展或可执行 audit retention。当前工作树已经实现这些 P2-06 能力，lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审也已通过；计划仍保持 active，因为独立 commit、clean-SHA Compose、push 和该精确 SHA 的远程 CI 尚未完成。 + +## Objective + +在不调用真实 Provider、不改变评分协议且不混入整库灾难恢复的前提下，交付可抓取、可告警、可验证的 P2-06：固定低基数 exporter、Worker DB-time progress/liveness、审计 retention archive/verify/restore/delete 维护工具与完整 Runbook，并以独立 commit、push 和精确 SHA 绿色 CI 闭环。 + +## Scope + +- 固定 Prometheus 文本 exposition endpoint，复用现有 DB current/history facts，并增加 Worker progress/stalled facts。 +- Alert rules 与 Runbook，覆盖 backlog、dead-letter、governance integrity、overdraw、queue degraded、Worker stalled 和恢复持续时间。 +- Worker process/progress 持久模型、Alembic、repository、Worker 主循环接线和只读 API/schema。 +- Audit retention archive、verify、restore、delete CLI，内容 hash/rollup、原子输出、严格路径/权限/失败语义。 +- SQLite 与 PostgreSQL 一致性、importer/schema fingerprint、Compose/config、API/安全/架构/部署/测试文档。 + +## Non-goals + +- Prometheus/Alertmanager/OTel 服务部署、通知发送器、登录/RBAC 或公网支持。 +- PostgreSQL+keyring 完整 backup/restore、Redis disaster recovery、PITR/RPO/RTO；这些属于 P2-07。 +- 真实 Provider 性能、费用、exactly-once、HA 或生产 SLA。 +- 高基数标签、任意日志转指标、raw audit 导出 API、自动请求链路删除或 WORM 声明。 +- Benchmark/协议/评分/前端产品功能变化。 + +## Assumptions + +- 固定 exposition renderer 足以支持当前有限指标集，避免新增生产依赖；实现前由 ADR-0015 冻结。 +- Worker progress 需要新表，旧 Run/Worker 不回填虚构事实；多 Worker 仍只在 PostgreSQL 正式支持。 +- Archive 是显式受控 CLI 产物，存放在仓库外/ignored 路径并作为敏感运维数据保护。 +- DB current timestamp 是 progress、stale、cutoff 与 retention 的唯一时间源。 + +## Requirements + +- `docs/NEXT_TASK.md` P2-06 全部条目。 +- AGENTS §3.2 的数据库/API/安全/文档联动与 §4 验证要求。 +- ADR-0005 的数据库权威任务状态与 metrics 非权威原则。 +- ADR-0009 的 typed audit、90/365 天 minimum retention、先 archive/verify/rollup 后 cleanup 及非 WORM 边界。 +- ADR-0010 的 Run latency 数据来源与不扩大 Provider/credential audit 数据面。 + +## Implementation steps + +1. [completed] 勘察并冻结 ADR-0015 合同。 + - Files/modules: `backend/app/api/v1/health.py`、`schemas/system.py`、`workers/service.py`、`worker_probe.py`、`models/governance.py`、`governance/audit.py`、migration/importer、相关文档。 + - Validation: 记录现状/竞态/安全边界；ADR 明确指标名/标签、DB 压力、Worker stale、archive format/cutoff/commit outcome 和 rollback。 +2. [completed] 实现 Worker progress schema、持久写入与读模型。 + - Files/modules: models、Alembic `0005`、repository/service/probe、schemas/API、migration/importer。 + - Result: generation 注册/停止、DB UTC 合并刷新、四类真实 event、inclusive live cutoff、JSON 聚合、`0005` guard 与 13 表 importer 已实现；importer committed-target postverify 使用同一 canonical integrity contract；probe 固定为 capability-only。 + - Validation: DB UTC、节流、启动/scan/claim/progress/heartbeat/stop/stale、并发/错误路径目标回归与全量回归已通过；完整计数见 Validation。 +3. [completed] 实现受控 exporter 与告警规则。 + - Files/modules: health/observability API、Prometheus renderer、规则文件、规则验证测试、Compose/config。 + - Result: 固定 text `0.0.4` gauge、单 DB-time snapshot、audit/latency hard cap、single-flight、固定八规则/抓取示例与 Runbook 已实现；取消中的 collection 在同步 DB 工作结束前继续持有 gate。 + - Validation: 固定 content type/name/labels、audit/DB/renderer fail closed、取消竞态、无对象 ID/secret label和规则合同目标/全量回归已通过；Compose 告警链路仍属步骤 6。 +4. [completed] 实现 audit retention 维护工具。 + - Files/modules: audit validator、maintenance CLI、schemas/manifest、双方言测试。 + - Result: canonical archive v1、原子 no-replace `0600` 输出、真正离线 verify、精确 reconcile/restore/delete、默认不删除和 exit 2/3/4 已实现；`app.db`/`app.governance` 的 runtime export 改为 lazy，离线 verify 不构造 engine。 + - Validation: strict JSON/size/path/permission/duplicate-key/hash、冻结 fixture、SQLite/PG round-trip、same-key/different-fact 与 commit outcome 回归已通过。 +5. [completed] 完成文档、迁移/importer、运行与安全联动。 + - Files/modules: README、API、Architecture、Security、Testing、Deployment、Operations、Compose/env、prepare/importer。 + - Result: 相关代码/Compose/env 及主要协议、API、安全、架构、部署、运维和状态文档已进入当前 diff；状态、测试、计划和工作日志已同步。 + - Validation: 本轮九份状态文档相对链接检查与 `git diff --check` 通过；最终全仓规则/秘密审计属于步骤 6。 + - Logging: 生产 logger 消息必须为无格式参数的字面量；第三方动态消息统一固定化，Uvicorn raw access log 禁用。 +6. [in_progress] 执行完整验证、终审和远程门禁。 + - Files/modules: 全部本切片文件与状态文档。 + - Validation: 定向+全量+lint+smoke+双方言 migration/PG integration+Compose/规则/secret/diff；dirty capacity/acceptance 与修复后 staged 技术/安全终审已通过；独立 commit 后 clean-SHA Compose、push、精确 SHA CI 4/4。 + +## Risks + +| 风险 | 可能性/影响 | 预防措施 | 触发后的处理 | +|---|---|---|---| +| 高频 scrape 放大数据库压力 | 中/高 | 固定有界窗口、样本上限、查询快照、抓取间隔文档 | fail whole scrape，降低频率并调查查询计划 | +| label/archive 泄密或高基数 | 中/高 | exact allowlist、无对象 ID label、固定错误、严格 archive schema | 停止发布，保留本地证据并修复后重建 archive | +| Worker progress 写放大/幽灵健康 | 中/高 | DB UTC、节流写、process generation、stale/stop 语义 | fail closed 标 stale，不让 dependency probe 覆盖 progress | +| archive 删除竞态/提交结果未知 | 低/高 | snapshot+cutoff+manifest hash 绑定、默认 archive-only、独立 verify/delete | 禁止盲重试，按 count/hash 只读对账 | +| schema/importer 历史兼容漂移 | 中/高 | 0005 migration fingerprint、双方言与 12→13 表 importer 测试 | 保持服务停止，向前修复或用已验证备份 | +| P2-06/P2-07 范围混合 | 中/中 | audit restore 只验证 archive 自身；整库/keyring 留待下一任务 | 更新计划并拆分，不以局部恢复宣称 DR | + +## Validation + +| 验收项 | 命令或检查 | 预期结果 | 实际结果 | +|---|---|---|---| +| Worker progress/API/exporter | 目标 pytest | DB UTC、stale、低基数、fail-closed 全通过 | 合并定向套件全绿；随后包含于后端 `916 passed, 33 skipped` 全量结果 | +| Audit retention | 目标 pytest + CLI self-check | archive/verify/restore/delete 与失败路径全通过 | 合并定向套件全绿；包括 no-op postverify、FIFO/line-cap 和 PG advisory/row-lock，随后包含于后端全量和 `33 passed, 0 skipped` 真实 integration | +| Migration/importer | SQLite + PostgreSQL integration | 0005/schema fingerprint/import 对账通过 | 临时 SQLite upgrade head/check 与真实 PG 往返/check 全绿；默认用户 SQLite 尚未到 head，直接 `alembic check` 按预期失败且未被擅自迁移；真实 importer 路径包含于 33 个 integration 用例 | +| 真实 PostgreSQL/Redis | `pytest -m integration` | 新增 retention 与既有 lease/governance/importer 全部收集且零 skip | 临时 PG16/Redis7 migration/check 后 `33 passed, 0 skipped`；首次 cleanup 被安全策略拒绝且未启动容器，改用明确目标后通过 | +| Lint/typecheck | `make lint` | Ruff、format check、ESLint、TypeScript 全通过 | 全绿；Ruff 检查 152 files，ESLint 与 TypeScript typecheck 通过 | +| Full test | `make test` | 后端/前端全量通过且只用 Mock/Stub | 后端 `916 passed, 33 skipped`；前端 `38 passed` | +| Offline smoke | `make smoke` | 只用 Mock 的最小垂直链路通过 | `1 passed, 7 deselected` | +| Frontend build | `cd frontend && npm run build` | production build 成功 | 成功，2192 modules；662.39 kB 主 chunk warning 为非阻断既有告警。首次在仓库根误运行 `npm run build` 因无 `package.json` 失败，随后使用正确目录通过 | +| Deployment | `docker compose config --quiet` | exit 0 | exit 0 | +| Alert rules | unit test + `promtool check rules` | 八条规则可被 Prometheus 解析且只引用已交付指标 | 临时 `prom/prometheus:v3.5.0` 验证八条规则全部成功 | +| Dirty Compose acceptance | `make phase2-acceptance` | 9/9、Worker `2/2/2/0/0`、两级 populated refusal/两层空库往返、cleanup empty | 通过；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | +| Dirty Compose capacity | `make phase2-capacity` | 最新脚本全通过且 cleanup empty | artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`；18/270/270/271/1229，0 error/drift/duplicate/PEL/lag，Worker expected 2，cleanup C/V/N/image=0；offline Mock、非 SLO | +| Scripts Ruff | 目标 correctness/import 规则 | P2-06 脚本无 E/F/I 问题 | 过宽默认命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | +| Staged 技术/安全终审 | Blocker/High/Medium 终审 + 受影响回归 | 无未解决 B/H/M | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | +| Docs/security review | Markdown 链接、secret/diff scan | 无坏链接、秘密或虚假状态 | 状态文档相对链接、whitespace diff、staged secret/path/blob 扫描均通过 | +| Remote gate | push 后 GitHub Actions exact SHA | 四个必需 job 全 success | 待执行 | + +## Rollback + +先停止 API/Worker/exporter scrape 与 retention maintenance。代码可关闭新增 exporter/Worker progress 写入，但保留新 schema 与已有 progress/archive 事实；只有新表为空且无依赖时才允许 0005 downgrade。Audit delete 默认关闭且永不自动运行；已经生成的 archive 先验证/hash 后按敏感文件策略保留。若 delete commit outcome unknown，禁止重跑，先在只读事务对账 event key/count/hash。整库恢复与 keyring 不在本切片回滚声明内。 + +## Documentation updates + +- [x] README / 用户操作说明已同步当前实现与 pending-gate 状态 +- [x] API / Schema 已进入当前 P2-06 diff +- [x] Architecture / Security / ADR-0015 已进入当前 P2-06 diff +- [x] Testing / Deployment / Operations / alert Runbook 已进入当前 P2-06 diff +- [x] CHANGELOG、PROJECT_STATUS、Roadmap、Phase 2、NEXT_TASK、工作日志已同步；最终门禁结果仍须在步骤 6 回填 + +## Completion evidence + +- Changed files: `0005`/models/importer/prepare、Worker service/runner/progress/probe、JSON/Prometheus API 与 collector/renderer、audit validator/archive/retention CLI、logging/config/Compose、规则与相关测试/文档。 +- Commands run: `make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 与 integration `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、正确目录 frontend build、`docker compose config --quiet`、Prometheus `v3.5.0` 八规则校验、dirty capacity 和 9/9 acceptance 成功。 +- Acceptance evidence: dirty artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker `2/2/2/0/0`、两级 populated refusal、两层空库往返与 cleanup empty。最新 dirty capacity artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，0 error/drift/duplicate/PEL/lag 且 cleanup empty；offline Mock、非 SLO。 +- Not run: 当前尚未完成最终文档/秘密/diff 检查、clean-SHA Compose 和精确 SHA CI。默认用户 SQLite 未迁移到 head，按保护原则保留原状。 +- Known issues: P2-07 仍为下一独立切片；Phase 2 保持 `in_progress`。 + +## Decision and discovery log + +| 日期 | 类型 | 记录 | 影响/后续 | +|---|---|---|---| +| 2026-08-28 | discovery | 现有 JSON metrics/history 已有 DB snapshot 与 audit fail-closed，可作为 exporter 数据源；仓库没有 exporter/alert sender。 | 复用现有事实，不建立第二套状态。 | +| 2026-08-28 | discovery | Worker probe 只验证 DB/head/queue capability，不能证明主循环 scan/claim/progress。 | 新增 DB-time process progress，并保持 probe 语义分离。 | +| 2026-08-28 | discovery | `audit_events` 有 minimum expiry 与索引，但没有可执行 archive/verify/delete/restore。 | ADR-0015 必须冻结显式维护协议。 | +| 2026-08-28 | decision | ADR-0015 冻结固定 15 分钟/50,000 行 exporter、DB-time coalesced Worker progress、八条规则及 canonical JSONL retention 流程。 | 三条实现线可在不改变事实来源和 P2-07 边界的前提下并行。 | +| 2026-08-28 | implementation | `0005` 增加 Worker progress 与两个 audit 扫描索引；importer 从 12 表扩展到 13 表并拒绝 live generation。 | 当前 Alembic/archive compatible head 为 `20260828_0005`；历史 `0004` 证据保留不改写。 | +| 2026-08-28 | implementation | exporter 采用由 collection owner 持有的进程内 gate；request 重复取消不会在线程 DB query 仍运行时释放 single-flight。 | 避免取消竞态导致第二次 scrape 与未结束 query 重叠。 | +| 2026-08-28 | implementation | `app.db` 与 `app.governance` runtime exports 改为 lazy，CLI `verify` fresh process 不再读取 DB 配置或创建 engine/目录。 | 保持 archive verification 真正离线，同时保留既有 public imports。 | +| 2026-08-28 | review | importer committed-target integrity、retention no-op postverify、外部 logger extra、FIFO/line cap 与 PostgreSQL advisory/row-lock 门禁经终审补强并进入回归。 | 相关目标测试、全量与真实 integration 已通过。 | +| 2026-08-28 | gate | Prometheus v3.5.0 八规则、dirty capacity 与 9/9 acceptance 通过；structured-extra High 与 Worker `__main__` logger Medium 修复后，最新 staged 技术/安全终审为 0 Blocker/High/Medium。 | acceptance `11554c25ec2d` / `d5f058…9fde`；capacity `c6de062ab77e` / `4aeb827…8453`；clean-SHA 仍须重跑。 | +| 2026-08-28 | status | P2-06 功能与 dirty 本地门禁已完成，但 commit/clean-SHA Compose/push 与精确 SHA CI 尚未完成。 | P2-06/Phase 2 均保持 `in_progress`；先完成步骤 6，再启动 P2-07。 | diff --git a/docs/worklogs/2026-08-28-phase-2-observability-retention.md b/docs/worklogs/2026-08-28-phase-2-observability-retention.md new file mode 100644 index 0000000..a737fb1 --- /dev/null +++ b/docs/worklogs/2026-08-28-phase-2-observability-retention.md @@ -0,0 +1,158 @@ +# 2026-08-28 — Phase 2 可观测性与审计保留工作日志 + +## 元信息 + +- 日期：2026-08-28 +- 执行者：Codex +- 分支：`codex/complete-evaluation-workflow` +- 初始 HEAD：`7cb44bfda2b0e53f88412a5346a0705799174df6` +- 关联阶段：[Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- 关联计划：[Phase 2 可观测性与审计保留执行计划](../plans/2026-08-28-phase-2-observability-retention.md) +- 决策基础：[ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](../decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) +- 本轮决策：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) +- 当前状态：`in_progress`；P2-06 单独推进，Phase 2 仍为 `in_progress` + +## 目标与背景 + +P2-01 已在精确提交与远程 CI 上闭环，现有系统也已经提供 PostgreSQL 派生的 `/tasks/metrics`、同一读快照内验证审计完整性的 `/tasks/history`、稳定分页的 Run audit、应用 JSON 日志和依赖 capability probe。本任务完成 P2-06：给这些事实增加低基数、fail-closed 的 scrape 出口和告警规则；增加由数据库 UTC 记录的 Worker 主循环进展，使依赖探针不再被误作执行活性；交付显式、可校验、可恢复且默认不删除的 audit retention 维护流程。 + +本切片完成后仍不宣称生产监控平台、不可篡改审计、HA 或灾难恢复完成。PostgreSQL 与数据库外 keyring 的配对 backup/restore、Redis 重建和完整故障矩阵保留给独立 P2-07。 + +## 初始状态与已保护边界 + +- 开始时工作树干净，当前分支与 `origin/codex/complete-evaluation-workflow` 同步。 +- 当前公开 API 没有认证，只允许可信 loopback；新增 exporter 与 Worker 状态不得扩大公网支持边界。 +- 数据库仍是任务、租约、审计和进展时间的唯一权威；Redis、进程内计数和 exporter 都不是第二事实来源。 +- 自动化只可使用 Mock/Stub，不调用真实或付费 Provider。 +- 审计 archive 不得包含 Key、Authorization、Cookie、credentialed DSN、ciphertext、nonce、URL、题目、Prompt、Response 正文或 Provider raw body，也不得称为 WORM。 +- 不修改 `llmbenchlab-protocol-v1`、Benchmark schema、评分或排行榜隔离。 + +## 范围 + +- Prometheus 文本 exposition 的受控 exporter：复用现有 DB gauges、typed counters、Run latency 与 Worker progress，固定指标名、固定 label 集合、采样窗口和错误语义。 +- 低基数告警规则与 Runbook：backlog、dead-letter、governance integrity、overdraw、queue degraded、Worker stalled 与恢复时长；记录 severity、`for`、silence 和响应动作。 +- Worker 主循环 DB-time progress：进程注册、最近 scan/claim/progress/heartbeat、停止与 stale 判定；依赖 probe 继续只表示 capability。 +- Audit retention 维护工具：严格 archive schema、原子写、内容 hash/rollup、独立 verify、幂等 restore、显式 delete-after-verify 与 commit outcome 边界。 +- Alembic/SQLite→PostgreSQL importer、API/schema、Compose/config、测试与所有相关文档联动。 + +## 非目标 + +- 不交付 Prometheus server、Alertmanager、OTel collector、通知发送器或公网监控。 +- 不交付生产 HA、PITR、RPO/RTO、PostgreSQL+keyring 完整 backup/restore 或 Redis 灾备；这些属于 P2-07。 +- 不新增真实 Provider 指标、Model/Run/question/provider/worker ID label 或用户控制字符串 label。 +- 不自动从 API 请求链路清理 audit，不把 archive 上传到对象存储，也不宣称不可篡改存储。 +- 不提前进入 Phase 3。 + +## 验收标准 + +1. Exporter 只输出固定低基数指标；数据库或 retained audit 完整性失败时整个 scrape 失败且不回显损坏值，不返回部分成功。 +2. 告警规则可由测试解析，全部引用已交付指标，并对持续时间、级别、静默和 Runbook 给出确定合同。 +3. Worker 主循环事实全部使用数据库 UTC，scan/claim/progress 与 lease heartbeat 明确分离；stalled 判断能识别“依赖正常但主循环不推进”。 +4. Retention archive/verify/restore/delete 在 SQLite 与 PostgreSQL 语义一致，默认只归档不删除；删除前必须验证同一 archive、稳定 snapshot 与 cutoff，commit unknown 不可盲目重试。 +5. Archive 与 exporter 的 schema、日志、错误和证据均通过秘密/高基数审查；无真实 Provider 调用。 +6. 目标测试、全量测试、lint、smoke、迁移、Compose/规则自检通过；独立 commit push 后该精确 SHA 的四个必需 CI job 全绿。 + +## 假设 + +- Prometheus 文本 exposition 可用项目内小型固定 renderer 实现，无需新增生产依赖；若勘察证明不安全，将在 ADR 中记录依赖取舍。 +- Worker progress 需要新的持久表和 Alembic revision；旧数据库在 upgrade 后不回填虚构进展。 +- Audit archive 是内部敏感运维文件；公开 API 不直接提供下载或删除能力。 +- Retention cutoff 由数据库 UTC 产生，archive 以 `expires_at < cutoff` 的稳定有界集合为目标；边界细节由 ADR 冻结。 + +## 风险 + +| 风险 | 影响 | 预防与处置 | +| --- | --- | --- | +| Exporter 每次抓取放大 DB 压力 | 高频 scrape 影响 Worker | 固定窗口/样本上限、最小 scrape 间隔文档、单快照查询、失败不重试风暴 | +| label 或 archive 泄露身份/秘密 | 长期外泄 | 指标无对象 ID label；archive 严格 allowlist、权限与原子写；错误固定化 | +| Worker progress 写放大或幽灵在线 | 错告警/数据库压力 | DB UTC、节流 upsert、进程启动/停止事实、stale 阈值与清理策略 | +| archive 后删除竞态或 commit unknown | 证据丢失/重复操作 | 归档、验证、删除分阶段；snapshot/cutoff/hash 绑定；删除事务与不确定结果 fail closed | +| 与 P2-07 交叉扩大范围 | 难审查/虚假完成 | 本切片只验证 audit archive 自身恢复；整库与 keyring 恢复另立任务 | + +## 实施步骤 + +1. 完成现有 API、Worker、审计、migration/importer 与运维文档勘察。 +2. 新增 ADR-0015，冻结 exporter、Worker progress、alert 与 retention/archive 合同。 +3. 实现 schema/migration、Worker progress repository/接线与读 API/exporter。 +4. 实现规则文件、Runbook 与规则校验测试。 +5. 实现 audit archive/verify/restore/delete CLI 及双方言/失败路径测试。 +6. 更新 importer、Compose/config、API、安全、架构、部署、测试和运维文档。 +7. 运行定向、全量、lint、smoke、migration、Compose 和安全门禁；修复终审问题。 +8. 更新状态文档，形成独立 commit，push 并等待精确 SHA CI。 + +## 初始勘察事实 + +- `GET /api/v1/tasks/metrics` 已有当前 DB gauges；`GET /api/v1/tasks/history` 已在一致读快照中验证 retained audit 后输出 typed counters 与三类 Run latency。 +- 当前文档明确写明“没有 Prometheus exporter、告警发送器或 Worker 主循环 liveness”；`worker_probe` 只检查 DB/head/queue capability。 +- `audit_events` 已有 `retention_class`、`occurred_at` 与 `expires_at` 及 expiry 索引，但没有 archive/verify/delete/restore 维护命令。 +- 本任务开始尚未修改生产代码、schema 或公共 API。 + +## 已实现结果（尚待仓库级门禁） + +### Worker progress、migration 与 importer + +- 新增 Alembic `20260828_0005` 与 `worker_processes` generation 表，字段/约束覆盖 DB UTC start/seen/scan/claim/progress/lease-heartbeat/stop；辅助索引为 `(stopped_at,last_seen_at,generation_id)`，audit 另增加 `(expires_at,id)` 和 `(occurred_at,id)` 有界扫描索引。 +- 长运行 `WorkerService.run()` 在执行前注册 generation，按真实 event bit 最多每五秒合并一笔短事务；timer 无 event 时零写入，提交失败保留 bit，graceful stop 原子写 pending event 与 stop，stopped generation 的 late flush 被 CAS 拒绝。 +- Runner/Worker 已在 claim、scan、Response insert、durable transition、reaper progress 与 lease renewal 的提交后边界接线；`run_once()`、probe 和其他短命 CLI 使用 null observer，不虚构长运行 generation。 +- `/tasks/metrics` 增加 expected/registered/live/stalled/shortfall/stale-after 与五个最近 DB-time 聚合；边界 `last_seen_at == cutoff` 为 live，响应不含 generation/worker ID。probe 固定增加 `probe_scope=dependencies_only`、`main_loop_progress=not_checked`。 +- SQLite→PostgreSQL importer 从历史 12 表扩展为当前 13 表；live generation 在打开目标前 fail closed，stopped/stale rows 可精确复制。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，空表才允许进入原 `0004` guard。 +- Importer 完整性终审把 committed target 的 postverify 收敛到同一 canonical count/PK/content contract，避免仅凭行数或事务前快照接受目标事实漂移。 + +### Exporter、规则与日志 + +- 新增 `GET /api/v1/metrics/prometheus`，固定 Prometheus text `0.0.4`、全部 gauge、固定 enum label 与顺序；current/governance/Worker、15 分钟 typed-audit、1 小时 Run latency 共用一个 DB-time 读快照，Redis 只作快照外非权威 observation。 +- audit 最多读取 `50,001` 行并在超限时整次 `503`；每类 latency 最多 `10,001` 样本并显式报告 truncated；数据库、retained audit 或 renderer 损坏均整次 fail closed，不返回 last-good cache 或部分 exposition。 +- collection owner 持有进程内 single-flight gate，request cancellation 不能取消同步 DB thread 或提前释放 gate；重叠 scrape 立即 `429`。 +- `deploy/observability/` 已加入固定八条 Prometheus rules 与 `60s` scrape 示例；规则只有固定 `severity`/`component` labels、Runbook/silence annotations，不加入 Prometheus/Alertmanager 服务或 sender。 +- 生产 logger 调用由 AST 回归限制为无格式参数字面量，并静态核对登记 logger 与 literal event/result/error code 全集；JSON formatter 对每个 allowlisted extra 执行固定 enum、canonical UUID/Redis stream ID、HTTP method/route 与有限数值规范化，非法 ID/数值省略，未知字符串固定为 `unsupported`。Redis publish/delivery 同时拒绝非 UUID Run/correlation identity；外部 logger 的动态消息/identity 固定化且不能通过伪造 extra 注入，raw Uvicorn access handler 关闭。 + +### Audit retention + +- 收敛公共 retained-row read validator，API/history/exporter/archive/restore 对 event contract、原始 canonical payload、hash、identity、retention interval、timestamp 与 finite numeric 使用同一严格合同。 +- 新增 `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete`。archive 在一个 DB snapshot 取 cutoff，只归档 `expires_at < cutoff` 且最多 10,000 条，默认不删除。 +- `llmbenchlab-audit-archive-v1` 使用 canonical UTF-8 JSONL、完整行事实、rollup、domain-separated content hash 与整文件 SHA-256；在逐行 decode 前执行全局行数上限，并拒绝超限、duplicate key、非 canonical JSON、重复/乱序事实、未知 head、FIFO/非普通文件、symlink/非 owner/过宽权限或可由 group/other 写入的父目录。 +- 输出以同目录 `0600` 临时文件、fsync、no-replace install 和父目录 fsync 完成；verify/delete/restore 从同一个 descriptor 解析。hash 只表示误改检测/精确绑定，不是签名或 WORM。 +- delete/restore 要求用户确认同一 archive digest，只处理 archive 中的精确事实；冲突、缺引用或 mixed state fail closed。PostgreSQL mutation 保持 advisory/row lock，零行 no-op 也执行 postverify；exit `3` 表示提交后验证失败，exit `4` 表示 commit outcome unknown，二者都要求先 `reconcile` 而非盲重试。 +- `app.db` 与 `app.governance` package runtime exports 改为 lazy；fresh process `verify` 在无效数据库配置下仍不导入/构造 engine，也不创建数据库父目录，保持真正离线。 + +### 状态边界 + +- 以上功能已进入当前工作树，lint/test/smoke/integration/migration/build/config/rules 与 dirty capacity/9/9 acceptance 已通过；后续 staged security review 发现 structured logging High，继续复核又发现 `python -m app.worker` logger Medium，两项修复与定向回归均已落盘。最新 76-file staged 技术/安全终审重新收敛为 0 Blocker/High/Medium。独立 commit、clean-SHA Compose、push 与精确 SHA GitHub Actions 尚未完成，因此 P2-06 仓库级状态保持 `in_progress`。 +- P2-06 archive restore 只恢复 typed audit archive 自身。PostgreSQL+keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置与剩余故障矩阵仍属于 P2-07；Phase 2 继续 `in_progress`。 + +## 实际命令与结果 + +| 命令/检查 | 结果 | +| --- | --- | +| `git status --short --branch` | 工作树干净；分支与 origin 同步 | +| 阅读 README、AGENTS、PROJECT_STATUS、ROADMAP、Phase 2、NEXT_TASK、PLANS、ADR-0005/0009/0010 与相关 API/Security/Testing/Operations/Deployment/Architecture | 已完成初始只读勘察；实现前继续核对具体模块 | +| 三路只读设计审查：exporter/alerts、Worker progress、audit retention | 已完成；未修改生产代码；结论已冻结进 ADR-0015 | +| 全生产 logger source 静态勘察 | 应用 logger 共 47 个调用；现有消息为固定模板，但第三方 logger/Uvicorn access 可绕过应用字段 allowlist；已纳入 formatter 与 AST 回归范围 | +| Worker progress/API/runner/migration/importer 目标 pytest 与 Ruff 批次 | 实现过程中已执行并完成；随后由合并定向、全量与真实 integration 门禁覆盖 | +| Prometheus exporter/rules、audit archive/retention/CLI、logging 目标 pytest 与 Ruff 批次 | 实现过程中已执行并完成；取消竞态、离线 verify 和 audit scan index 审查修正后已加入回归，随后由合并定向/全量门禁覆盖 | +| Structured logging High / Worker entrypoint Medium 修复门禁 | 应用 logger/message/event/result/error code 全集静态登记，`python -m app.worker` 真实入口固定使用已登记 `app.worker` logger；HTTP method、UUID/Redis stream ID、level/timestamp/duration、queue delivery canary 与无 I/O Worker 入口回归共 `83 passed`；凭据日志回归 `55 passed`；随后启动独立 Redis 7 容器并以 `LLMBENCHLAB_TEST_REDIS_URL=redis://127.0.0.1:16379/15` 实跑 `tests/integration/test_redis_streams.py`，结果 `2 passed` 且容器由 trap 停止；Ruff check、Ruff format check 与 `git diff --check` 全绿。独立 reviewer 再跑 logging/source/queue `19 passed`，最新 staged 技术/安全终审 0 Blocker/High/Medium | +| 合并 P2-06 定向 pytest 套件 | 全绿；随后 `make test` 也已完成，不单独虚构定向测试数 | +| 真实 PostgreSQL/Redis migration/check + `integration` marker | 临时 PostgreSQL 16/Redis 7 全绿，`33 passed, 0 skipped`；覆盖 audit retention advisory/row-lock 与既有 lease/governance/importer 路径。首次 cleanup 命令被本地安全策略拒绝，容器尚未启动；改用明确目标后通过 | +| `make lint` | 全绿：Ruff 检查 152 files、Ruff format check、ESLint、TypeScript typecheck 均通过 | +| `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed`；自动化只使用 Mock/MockTransport/stub | +| `make smoke` | `1 passed, 7 deselected`；完全离线 Mock | +| 从仓库根运行 `npm run build` | 失败：根目录没有 `package.json`；这是命令目录错误，不是 frontend build 失败 | +| `cd frontend && npm run build` | 成功：2192 modules；主 chunk `662.39 kB` 产生既有非阻断 warning | +| 默认用户 SQLite 上 `cd backend && uv run alembic check` | 失败：该用户数据库尚未在 `20260828_0005` head；按保护用户数据原则未擅自运行 upgrade | +| 临时 SQLite head→`0001`→head + `alembic check`，以及隔离真实 PostgreSQL migration round-trip/check | 全绿；当前 head 为 `20260828_0005`，包含 populated downgrade guard/空库往返 | +| `docker compose config --quiet` | exit 0 | +| 临时 `prom/prometheus:v3.5.0` 容器中 `promtool check rules` | SUCCESS；八条规则全部通过 | +| 过宽 Ruff scripts 命令 | 报告 93 条既有 modernization 告警；随后按本任务静态合同运行 `--select E,F,I` 并通过，未修改范围外历史现代化问题 | +| `make phase2-acceptance`（dirty 工作树） | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`；application populated `0005` refusal、isolated populated `0004` refusal、两层空库往返、cleanup containers/volumes/networks empty | +| `make phase2-capacity`（最新 dirty 工作树） | artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`；18 Runs/270 Responses/270 QuestionExecutions/271 reservations（270 actual + 1 conservative）/1229 audit，0 error/drift/duplicate/PEL/lag，Worker expected 2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | +| 先前 code review 与 hydration/import integrity 回归 | 当时结论为 0 Blocker/High/Medium、目标集 `67 passed`；其后 staged security review 发现 structured logging High 与 Worker entrypoint Medium，均已修复并重新复核 | +| 最新 staged 技术/安全终审 | 76-file index 为 0 Blocker/High/Medium；独立 logging/source/queue `19 passed`，archive/retention/CLI/exporter/Docker-script `167 passed`；secret/path/blob/category 扫描无真实凭据或禁止产物 | +| README、TESTING、CHANGELOG、PROJECT_STATUS、ROADMAP、Phase 2、NEXT_TASK、计划/工作日志状态同步 | 已把实现事实与 pending-gate/P2-07 边界写入当前 diff；本轮链接/diff 检查通过 | +| 九份状态文档相对链接检查 | 全部目标存在 | +| 七份 tracked 状态文档 `git diff --check` + 新 plan/worklog 的 `git diff --no-index --check` | 无 whitespace error；no-index 命令因文件整体为新增返回差异状态 1，且没有 `--check` 诊断 | + +## 已知问题与下一步 + +- P2-06 三条实现线、structured logging High 与 Worker entrypoint Medium 修复已落地；lint、全量 test、smoke、frontend build、临时 SQLite/真实 PG migration、真实 PG/Redis integration、Compose config、八规则 promtool、dirty capacity/9/9 acceptance 与最新 staged 技术/安全终审均已通过。 +- 全门禁通过后形成独立 commit，在 clean SHA 重跑 capacity/acceptance，普通 push 当前分支并等待该精确 SHA 四个必需 CI job 全绿；在此之前不得写成 P2-06 completed。 +- P2-07 未开始，仍缺数据库+keyring 配对 backup/restore、Redis 重建、告警响应与完整恢复矩阵；Phase 2 继续 `in_progress`。 diff --git a/scripts/phase2_acceptance.py b/scripts/phase2_acceptance.py index cb417ae..b231c8b 100644 --- a/scripts/phase2_acceptance.py +++ b/scripts/phase2_acceptance.py @@ -23,10 +23,10 @@ import math import os import re +import secrets import shutil import signal import socket -import secrets import subprocess import sys import tempfile @@ -42,6 +42,7 @@ PROTOCOL_VERSION = "llmbenchlab-protocol-v1" PRE_GOVERNANCE_REVISION = "20260827_0003" GOVERNANCE_REVISION = "20260827_0004" +WORKER_PROGRESS_REVISION = "20260828_0005" TASK_MESSAGE_VERSION = "llmbenchlab-run-task-v1" TASK_STREAM = "llmbenchlab:runs:v1" TASK_GROUP = "llmbenchlab-workers-v1" @@ -413,6 +414,7 @@ def __init__(self, repository_root: Path, artifacts_root: Path) -> None: "LLMBENCHLAB_COMPOSE_WORKER_HEARTBEAT_SECONDS": "2", "LLMBENCHLAB_COMPOSE_WORKER_POLL_SECONDS": "0.15", "LLMBENCHLAB_COMPOSE_WORKER_SHUTDOWN_GRACE_SECONDS": "1", + "LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES": "2", "LLMBENCHLAB_COMPOSE_MOCK_GENERATION_DELAY_SECONDS": "0.35", "LLMBENCHLAB_COMPOSE_REDIS_OPERATION_TIMEOUT_SECONDS": "0.75", "LLMBENCHLAB_COMPOSE_DATABASE_POOL_TIMEOUT_SECONDS": "2", @@ -1152,6 +1154,66 @@ def psql_database( check=check, ) + def clone_application_database(self, target_database: str) -> Dict[str, Any]: + """Clone the application database without racing its healthcheck connection.""" + + self.require( + SAFE_ID_PATTERN.fullmatch(target_database) is not None, + "unsafe PostgreSQL clone database name", + ) + self.require( + target_database not in {"llmbenchlab", "postgres"}, + "refused to replace a protected PostgreSQL database", + ) + restoration: Optional[subprocess.CompletedProcess] = None + try: + self.psql_database( + 'ALTER DATABASE "llmbenchlab" WITH ALLOW_CONNECTIONS false;', + database="postgres", + ) + terminated_raw = self.psql_database( + "SELECT json_build_object(" + "'attempted', count(*), " + "'all_terminated', COALESCE(bool_and(terminated), true))::text " + "FROM (SELECT pg_terminate_backend(pid) AS terminated " + "FROM pg_stat_activity WHERE datname = 'llmbenchlab' " + "AND pid <> pg_backend_pid()) active_connections;", + database="postgres", + ).stdout.strip() + try: + terminated = json.loads(terminated_raw) + except json.JSONDecodeError as exc: + raise AcceptanceFailure( + "PostgreSQL returned invalid source connection termination evidence" + ) from exc + self.require( + terminated.get("all_terminated") is True, + "could not terminate every application database connection before clone", + terminated, + ) + created = self.psql_database( + f'CREATE DATABASE "{target_database}" TEMPLATE "llmbenchlab";', + database="postgres", + ) + finally: + restoration = self.psql_database( + 'ALTER DATABASE "llmbenchlab" WITH ALLOW_CONNECTIONS true;', + database="postgres", + check=False, + ) + self.require( + restoration.returncode == 0, + "failed to restore application database connections after clone", + redact_text(restoration.stderr or restoration.stdout), + ) + + return { + "source_connections_disabled": True, + "terminated_connection_count": int(terminated["attempted"]), + "source_connections_restored": restoration is not None, + "create_returncode": created.returncode, + } + def db_run_snapshot(self, run_id: str) -> Dict[str, Any]: self.require(SAFE_ID_PATTERN.fullmatch(run_id) is not None, "unsafe Run ID") sql = """ @@ -1448,6 +1510,27 @@ def wait_api_ready(self, expected_status: int = 200, timeout: float = 90) -> Dic interval=0.25, ) + def wait_worker_progress_healthy(self, timeout: float = 90) -> Dict[str, Any]: + """Wait for both configured Worker main loops to report current progress.""" + + result = self.wait_for( + "two registered and live Worker main loops without stalls or shortfall", + lambda: self.http_json("GET", "/tasks/metrics", accepted={200}, timeout=4), + lambda response: all( + response["payload"].get(field) == expected + for field, expected in { + "worker_expected_processes": 2, + "worker_registered_processes": 2, + "worker_live_processes": 2, + "worker_stalled_processes": 0, + "worker_shortfall_processes": 0, + }.items() + ), + timeout=timeout, + interval=0.25, + ) + return result["payload"] + def setup_stack(self) -> None: self.stack_touched = True self.compose( @@ -1526,6 +1609,7 @@ def topology_scenario(self) -> None: live = self.http_json("GET", "/live", accepted={200}) health = self.http_json("GET", "/health", accepted={200}) ready = self.wait_api_ready(200) + worker_progress = self.wait_worker_progress_healthy() info = self.http_json("GET", "/info", accepted={200}) frontend = self.http_json("GET", "/healthz", accepted={200}, frontend=True) self.require(live["payload"]["status"] == "live", "liveness payload changed", live) @@ -1553,6 +1637,7 @@ def topology_scenario(self) -> None: "live": live["payload"], "health": health["payload"], "ready": ready["payload"], + "worker_progress": worker_progress, "frontend_healthz": frontend["payload"], "fixtures": demo, } @@ -2202,7 +2287,7 @@ def running_cancel_scenario(self) -> None: def migration_round_trip_scenario(self) -> None: with self.scenario( - "postgres_populated_0004_downgrade_refusal_and_empty_round_trip" + "postgres_populated_0005_and_0004_downgrade_guards_with_empty_round_trips" ) as entry: self.require(self.baseline_run_id is not None, "baseline Run is unavailable") run_id = self.baseline_run_id @@ -2213,20 +2298,26 @@ def migration_round_trip_scenario(self) -> None: self.require(active_raw == "0", "migration round trip found active Runs", active_raw) core_before = self.db_core_protocol_snapshot(run_id) reliability_before = self.db_run_snapshot(run_id) - populated_counts = json.loads( + application_counts = json.loads( self.psql( "SELECT json_build_object(" "'policies', (SELECT count(*) FROM governance_policies), " "'reservations', (SELECT count(*) FROM provider_call_reservations), " - "'audit_events', (SELECT count(*) FROM audit_events))::text;" + "'audit_events', (SELECT count(*) FROM audit_events), " + "'worker_processes', (SELECT count(*) FROM worker_processes))::text;" ).stdout.strip() ) self.require( - populated_counts["policies"] > 0 - and populated_counts["reservations"] > 0 - and populated_counts["audit_events"] > 0, - "0004 downgrade refusal requires populated policy, ledger, and audit evidence", - populated_counts, + application_counts["worker_processes"] > 0, + "0005 downgrade refusal requires persisted Worker process facts", + application_counts, + ) + self.require( + application_counts["policies"] > 0 + and application_counts["reservations"] > 0 + and application_counts["audit_events"] > 0, + "0004 downgrade refusal requires populated governance evidence", + application_counts, ) self.compose("stop", "api", "worker", timeout=90) @@ -2244,45 +2335,196 @@ def migration_round_trip_scenario(self) -> None: stopped_workers, ) - populated_downgrade = self.compose( + worker_progress_downgrade = self.compose( "run", "--rm", "--no-deps", "migrate", "alembic", "downgrade", - PRE_GOVERNANCE_REVISION, + GOVERNANCE_REVISION, timeout=180, check=False, ) self.require( - populated_downgrade.returncode != 0, - "populated 0004 downgrade unexpectedly discarded governance evidence", + worker_progress_downgrade.returncode != 0, + "populated 0005 downgrade unexpectedly discarded Worker progress evidence", + ) + worker_progress_refusal_output = "\n".join( + part + for part in ( + worker_progress_downgrade.stdout, + worker_progress_downgrade.stderr, + ) + if part ) - refusal_output = populated_downgrade.stderr or populated_downgrade.stdout self.require( - "Cannot downgrade governance schema" in refusal_output, - "populated 0004 downgrade did not return the stable refusal reason", - redact_text(refusal_output[-2000:]), + "Cannot downgrade Worker progress schema" in worker_progress_refusal_output, + "populated 0005 downgrade did not return the stable refusal reason", + redact_text(worker_progress_refusal_output[-2000:]), ) - populated_revision = self.psql( + application_revision = self.psql( "SELECT version_num FROM alembic_version;" ).stdout.strip() self.require( - populated_revision == GOVERNANCE_REVISION, + application_revision == WORKER_PROGRESS_REVISION, "failed populated downgrade changed the application database revision", - populated_revision, + application_revision, ) core_after_refusal = self.db_core_protocol_snapshot(run_id) reliability_after_refusal = self.db_run_snapshot(run_id) + governance_database = "p2governance_" + self.project[-12:] + governance_database_url = ( + f"postgresql+psycopg://llmbenchlab:{LOCAL_PASSWORD}@postgres:5432/" + f"{governance_database}?connect_timeout=3" + ) + governance_guard: dict[str, Any] = { + "database": governance_database, + "source_application_revision": application_revision, + "worker_facts_cleared_in_clone_only": True, + } + governance_guard["safe_template_clone"] = self.clone_application_database( + governance_database + ) + try: + self.psql_database( + "TRUNCATE TABLE worker_processes;", + database=governance_database, + ) + prepare_governance = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={governance_database_url}", + "migrate", + "alembic", + "downgrade", + GOVERNANCE_REVISION, + timeout=180, + ) + governance_revision_before = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=governance_database, + ).stdout.strip() + self.require( + governance_revision_before == GOVERNANCE_REVISION, + "isolated populated database did not prepare revision 0004", + governance_revision_before, + ) + governance_counts_before = json.loads( + self.psql_database( + "SELECT json_build_object(" + "'policies', (SELECT count(*) FROM governance_policies), " + "'reservations', (SELECT count(*) FROM provider_call_reservations), " + "'audit_events', (SELECT count(*) FROM audit_events))::text;", + database=governance_database, + ).stdout.strip() + ) + self.require( + governance_counts_before["policies"] > 0 + and governance_counts_before["reservations"] > 0 + and governance_counts_before["audit_events"] > 0, + "isolated 0004 downgrade guard lacks populated governance evidence", + governance_counts_before, + ) + governance_downgrade = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={governance_database_url}", + "migrate", + "alembic", + "downgrade", + PRE_GOVERNANCE_REVISION, + timeout=180, + check=False, + ) + self.require( + governance_downgrade.returncode != 0, + "populated 0004 downgrade unexpectedly discarded governance evidence", + ) + governance_refusal_output = "\n".join( + part + for part in (governance_downgrade.stdout, governance_downgrade.stderr) + if part + ) + self.require( + "Cannot downgrade governance schema" in governance_refusal_output, + "populated 0004 downgrade did not return the stable refusal reason", + redact_text(governance_refusal_output[-2000:]), + ) + governance_revision_after = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=governance_database, + ).stdout.strip() + self.require( + governance_revision_after == GOVERNANCE_REVISION, + "failed populated 0004 downgrade changed the isolated database revision", + governance_revision_after, + ) + governance_counts_after = json.loads( + self.psql_database( + "SELECT json_build_object(" + "'policies', (SELECT count(*) FROM governance_policies), " + "'reservations', (SELECT count(*) FROM provider_call_reservations), " + "'audit_events', (SELECT count(*) FROM audit_events))::text;", + database=governance_database, + ).stdout.strip() + ) + self.require( + governance_counts_after == governance_counts_before, + "refused populated 0004 downgrade changed governance evidence", + { + "before": governance_counts_before, + "after": governance_counts_after, + }, + ) + governance_guard.update( + { + "prepare_0004_returncode": prepare_governance.returncode, + "revision_before_refusal": governance_revision_before, + "populated_counts_before": governance_counts_before, + "downgrade_to_0003_returncode": governance_downgrade.returncode, + "refusal_reason": "Cannot downgrade governance schema", + "revision_after_refusal": governance_revision_after, + "populated_counts_after": governance_counts_after, + } + ) + finally: + drop_governance = self.psql_database( + f'DROP DATABASE IF EXISTS "{governance_database}" WITH (FORCE);', + database="postgres", + check=False, + ) + governance_guard["drop_returncode"] = drop_governance.returncode + self.require( + governance_guard["drop_returncode"] == 0, + "isolated populated governance database cleanup failed", + governance_guard, + ) + application_worker_facts_after = int( + self.psql("SELECT count(*) FROM worker_processes;").stdout.strip() + ) + self.require( + application_worker_facts_after == application_counts["worker_processes"], + "isolated governance guard changed application Worker process facts", + { + "before": application_counts["worker_processes"], + "after": application_worker_facts_after, + }, + ) + empty_database = "p2roundtrip_" + self.project[-12:] empty_database_url = ( - "postgresql+psycopg://llmbenchlab:{}@postgres:5432/{}?connect_timeout=3" - ).format(LOCAL_PASSWORD, empty_database) - empty_round_trip: Dict[str, Any] = {"database": empty_database} + f"postgresql+psycopg://llmbenchlab:{LOCAL_PASSWORD}@postgres:5432/" + f"{empty_database}?connect_timeout=3" + ) + empty_round_trips: dict[str, Any] = {"database": empty_database} self.psql_database( - 'CREATE DATABASE "{}";'.format(empty_database), + f'CREATE DATABASE "{empty_database}";', database="postgres", ) try: @@ -2291,7 +2533,7 @@ def migration_round_trip_scenario(self) -> None: "--rm", "--no-deps", "-e", - "DATABASE_URL={}".format(empty_database_url), + f"DATABASE_URL={empty_database_url}", "migrate", "alembic", "upgrade", @@ -2303,8 +2545,8 @@ def migration_round_trip_scenario(self) -> None: database=empty_database, ).stdout.strip() self.require( - empty_head_before == GOVERNANCE_REVISION, - "empty PostgreSQL database did not upgrade to 0004", + empty_head_before == WORKER_PROGRESS_REVISION, + "empty PostgreSQL database did not upgrade to 0005", empty_head_before, ) downgrade_empty = self.compose( @@ -2312,7 +2554,71 @@ def migration_round_trip_scenario(self) -> None: "--rm", "--no-deps", "-e", - "DATABASE_URL={}".format(empty_database_url), + f"DATABASE_URL={empty_database_url}", + "migrate", + "alembic", + "downgrade", + GOVERNANCE_REVISION, + timeout=180, + ) + empty_pre_worker_progress = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=empty_database, + ).stdout.strip() + self.require( + empty_pre_worker_progress == GOVERNANCE_REVISION, + "empty PostgreSQL database did not downgrade across 0005", + empty_pre_worker_progress, + ) + reupgrade_worker_progress = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={empty_database_url}", + "migrate", + "alembic", + "upgrade", + WORKER_PROGRESS_REVISION, + timeout=180, + ) + worker_progress_head_after = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=empty_database, + ).stdout.strip() + self.require( + worker_progress_head_after == WORKER_PROGRESS_REVISION, + "empty PostgreSQL database did not complete the 0005 round trip", + worker_progress_head_after, + ) + + prepare_empty_governance = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={empty_database_url}", + "migrate", + "alembic", + "downgrade", + GOVERNANCE_REVISION, + timeout=180, + ) + empty_governance_before = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=empty_database, + ).stdout.strip() + self.require( + empty_governance_before == GOVERNANCE_REVISION, + "empty PostgreSQL database did not prepare revision 0004", + empty_governance_before, + ) + downgrade_empty_governance = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={empty_database_url}", "migrate", "alembic", "downgrade", @@ -2328,48 +2634,83 @@ def migration_round_trip_scenario(self) -> None: "empty PostgreSQL database did not downgrade across 0004", empty_pre_governance, ) - reupgrade_empty = self.compose( + reupgrade_empty_governance = self.compose( + "run", + "--rm", + "--no-deps", + "-e", + f"DATABASE_URL={empty_database_url}", + "migrate", + "alembic", + "upgrade", + GOVERNANCE_REVISION, + timeout=180, + ) + governance_head_after = self.psql_database( + "SELECT version_num FROM alembic_version;", + database=empty_database, + ).stdout.strip() + self.require( + governance_head_after == GOVERNANCE_REVISION, + "empty PostgreSQL database did not complete the 0004 round trip", + governance_head_after, + ) + final_reupgrade_and_check = self.compose( "run", "--rm", "--no-deps", "-e", - "DATABASE_URL={}".format(empty_database_url), + f"DATABASE_URL={empty_database_url}", "migrate", "sh", "-c", "alembic upgrade head && alembic check", timeout=180, ) - empty_head_after = self.psql_database( + empty_final_head = self.psql_database( "SELECT version_num FROM alembic_version;", database=empty_database, ).stdout.strip() self.require( - empty_head_after == GOVERNANCE_REVISION, - "empty PostgreSQL database did not return to 0004", - empty_head_after, + empty_final_head == WORKER_PROGRESS_REVISION, + "empty PostgreSQL database did not return to migration head", + empty_final_head, ) - empty_round_trip.update( + empty_round_trips.update( { - "upgrade_returncode": upgrade_empty.returncode, - "head_before": empty_head_before, - "downgrade_returncode": downgrade_empty.returncode, - "pre_governance_revision": empty_pre_governance, - "reupgrade_and_check_returncode": reupgrade_empty.returncode, - "head_after": empty_head_after, + "initial_upgrade_returncode": upgrade_empty.returncode, + "initial_head": empty_head_before, + "worker_progress_0005_to_0004_round_trip": { + "downgrade_returncode": downgrade_empty.returncode, + "pre_worker_progress_revision": empty_pre_worker_progress, + "reupgrade_returncode": reupgrade_worker_progress.returncode, + "head_after_reupgrade": worker_progress_head_after, + }, + "governance_0004_to_0003_round_trip": { + "prepare_0004_returncode": prepare_empty_governance.returncode, + "head_before": empty_governance_before, + "downgrade_returncode": downgrade_empty_governance.returncode, + "pre_governance_revision": empty_pre_governance, + "reupgrade_returncode": reupgrade_empty_governance.returncode, + "head_after_reupgrade": governance_head_after, + }, + "final_reupgrade_and_check_returncode": ( + final_reupgrade_and_check.returncode + ), + "final_head": empty_final_head, } ) finally: drop_empty = self.psql_database( - 'DROP DATABASE IF EXISTS "{}" WITH (FORCE);'.format(empty_database), + f'DROP DATABASE IF EXISTS "{empty_database}" WITH (FORCE);', database="postgres", check=False, ) - empty_round_trip["drop_returncode"] = drop_empty.returncode + empty_round_trips["drop_returncode"] = drop_empty.returncode self.require( - empty_round_trip["drop_returncode"] == 0, + empty_round_trips["drop_returncode"] == 0, "isolated empty migration database cleanup failed", - empty_round_trip, + empty_round_trips, ) hashes = { @@ -2382,11 +2723,17 @@ def migration_round_trip_scenario(self) -> None: "active_runs_before": 0, "api_stopped": stopped_api, "workers_stopped": stopped_workers, - "populated_counts": populated_counts, - "populated_downgrade_returncode": populated_downgrade.returncode, - "populated_downgrade_refusal": "Cannot downgrade governance schema", - "application_revision_after_refusal": populated_revision, - "empty_database_round_trip": empty_round_trip, + "application_worker_progress_0005_to_0004_guard": { + "populated_counts": application_counts, + "downgrade_returncode": worker_progress_downgrade.returncode, + "refusal_reason": "Cannot downgrade Worker progress schema", + "revision_after_refusal": application_revision, + "worker_facts_after_isolated_governance_guard": ( + application_worker_facts_after + ), + }, + "isolated_governance_0004_to_0003_guard": governance_guard, + "empty_database_round_trips": empty_round_trips, "core_protocol_hashes": hashes, "core_snapshot_before": core_before, "reliability_before": reliability_before, diff --git a/scripts/phase2_capacity.py b/scripts/phase2_capacity.py index ac56ee8..60c792b 100755 --- a/scripts/phase2_capacity.py +++ b/scripts/phase2_capacity.py @@ -1079,6 +1079,7 @@ def __init__(self, repository_root: Path, args: argparse.Namespace) -> None: self.env["LLMBENCHLAB_COMPOSE_WORKER_LEASE_SECONDS"] = str(self.lease_seconds) self.env["LLMBENCHLAB_COMPOSE_WORKER_HEARTBEAT_SECONDS"] = str(self.heartbeat_seconds) self.env["LLMBENCHLAB_COMPOSE_WORKER_POLL_SECONDS"] = str(self.worker_poll_seconds) + self.env["LLMBENCHLAB_COMPOSE_WORKER_EXPECTED_PROCESSES"] = str(self.worker_count) self.env["LLMBENCHLAB_COMPOSE_WORKER_MAX_ATTEMPTS"] = str(self.worker_max_attempts) self.env["LLMBENCHLAB_COMPOSE_WORKER_RETRY_BACKOFF_BASE_SECONDS"] = str( self.retry_backoff_base_seconds From ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Fri, 28 Aug 2026 19:04:23 +0800 Subject: [PATCH 02/12] docs: record phase 2 observability evidence --- CHANGELOG.md | 6 +- README.md | 10 +-- docs/NEXT_TASK.md | 64 ++++--------------- docs/PROJECT_STATUS.md | 39 +++++------ docs/ROADMAP.md | 23 +++---- docs/TESTING.md | 24 ++++--- docs/phases/PHASE-2-RELIABILITY.md | 39 +++++------ ...6-08-28-phase-2-observability-retention.md | 25 +++++--- ...6-08-28-phase-2-observability-retention.md | 13 ++-- 9 files changed, 114 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf635b7..c244b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Added -- Added the in-progress P2-06 implementation under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. +- Added the P2-06 implementation, now `pending_on_docs_ci`, under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. - Added Alembic revision `20260828_0005`, the `worker_processes` generation table, bounded `(expires_at,id)` / `(occurred_at,id)` audit scan indexes, and DB-time coalesced Worker scan/claim/lease-heartbeat/progress recording. `/tasks/metrics` and the exporter expose only low-cardinality aggregates; the dependency probe remains capability-only. The stopped SQLite-to-PostgreSQL importer now copies and reconciles all 13 application tables, rejects a still-live Worker generation, and preserves stopped/stale process facts. - Added `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete`. The v1 canonical JSONL archive is bounded, exact-digest bound, atomically installed without replacement at `0600`, and independently verifiable without a database engine; delete is never implicit, restore/delete require an exact confirmed archive SHA-256, and commit-unknown outcomes require read-only reconciliation before retry. These integrity hashes are neither signatures nor WORM guarantees. - Added the Phase 2 single-host control-plane qualification under ADR-0012 through ADR-0014. The `P2-local-control-plane-v2` harness fixes one warm-up plus exactly five measured trials, four Mock-only capacity cells, paired Worker scaling, queue/execution/end-to-end and recovery objectives, ledger-derived correctness gates, bounded capacity formulas, and scoped project-image cleanup. Clean implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` passed 23/23 SLO checks with all six child invariants and cleanup checks; the earlier v1 aggregate remains permanently `unqualified` rather than being discarded or reclassified. This qualifies only the recorded single-host Mock control-plane profile, not production SLA, real-Provider capacity, HA, billing, or external exactly-once behavior. @@ -107,11 +107,11 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Verification -- P2-06 combined targeted regression suites are green. On the current uncommitted worktree, `make lint` passed Ruff across 152 files plus ESLint and TypeScript typecheck; `make test` passed with `916 passed, 33 skipped` on the backend and `38 passed` on the frontend; offline Mock smoke passed `1 passed, 7 deselected`; a temporary PostgreSQL 16/Redis 7 environment passed migration/check and `33 passed, 0 skipped` integration; temporary SQLite head-to-`0001`-to-head migration/check, the frontend production build, Compose config, and Prometheus `v3.5.0` `promtool check rules` for all eight rules are green. Dirty-tree Compose acceptance passed 9/9 with evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`), Worker gauges `2/2/2/0/0`, both populated downgrade refusals, two empty-database round-trips, and empty container/volume/network cleanup. The latest dirty capacity also passed with evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`): 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit events, zero error/drift/duplicate/PEL/lag, expected Workers 2, and empty container/volume/network/image cleanup; it is an offline Mock observation, not an SLO. Code review reported 0 Blocker/High/Medium; the later staged-security structured-extra reflection and `python -m app.worker` logger findings were fixed, and the latest 76-file staged technical/security review again reports 0 Blocker/High/Medium. Independent commit/push, clean-SHA Compose evidence, and exact-SHA GitHub Actions are still pending. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. This does not mark P2-06 repository closeout or Phase 2 complete. +- P2-06 implementation gates are complete. Commit [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) was pushed to `codex/complete-evaluation-workflow` and is tracked by [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3); exact-SHA GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) passed all four required jobs. Its local gates include green combined targeted suites, `make lint` across 152 Python files plus ESLint/TypeScript, backend `916 passed, 33 skipped`, frontend `38 passed`, offline Mock smoke `1 passed, 7 deselected`, PostgreSQL 16/Redis 7 migration/check and `33 passed, 0 skipped` integration, isolated SQLite migration/check, frontend build, Compose config, eight-rule Prometheus `v3.5.0` validation, and a 76-file technical/security review with 0 Blocker/High/Medium. Clean-SHA Compose acceptance passed 9/9 with `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` (SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`), Worker gauges `2/2/2/0/0`, and empty container/volume/network cleanup. Clean capacity passed with `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` (SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`): 1W/2W/burst QPS `7.267474/12.962228/9.333604`, wall `8.255963/4.628834/6.428385s`, 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit events, zero question error/drift/duplicate/PEL/lag, Worker expected 2 with shortfall 0, and empty container/volume/network/image cleanup with image counters `1/1/0/0`. The earlier dirty acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`) and dirty capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`, 1229 audit events) remain historical evidence rather than being overwritten. These are offline Mock observations, not production or real-Provider SLOs. P2-06 is `pending_on_docs_ci`, not complete: only this evidence-documentation commit and its own exact-SHA CI remain. Phase 2 stays `in_progress`, and P2-07 is `not_started`. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. - Recorded two corrected local gate invocations without hiding their first outcomes: the initial integration cleanup command was rejected by the local safety policy before any container started, then the explicit cleanup/run passed; an over-broad Ruff invocation over `scripts/` surfaced 93 existing modernization warnings, while the intended `--select E,F,I` correctness/import gate passed. - On the governance/audit candidate at `665244e095905083b606b8e98e946ed1a02dc0fc`, `make test` passed with `604 passed, 29 skipped` on the backend and `38 passed` on the frontend; a separate real PostgreSQL/Redis integration run passed `29/29`. That frozen implementation also passed `make lint`, offline `make smoke` (`1 passed, 7 deselected`), the frontend production build, isolated SQLite and PostgreSQL migration/check gates, `docker compose config --quiet`, targeted governance/API/Worker suites, enhanced capacity, and 9/9 Compose acceptance. - Pushed the implementation and lock-order fixes through exact SHA `665244e095905083b606b8e98e946ed1a02dc0fc`. Its enhanced capacity evidence (`40deadeb…0588`) records finite policy, exact 4×`202` + 2×`429`, cooperative/cross-Model fairness, fault recovery, 18 Runs/270 Responses/271 reservations/1229 audit events, and empty cleanup; its 9/9 acceptance evidence (`ab311665…ddec`) includes all three deterministic database seams and empty cleanup. GitHub Actions run `33099260233` passed all four required jobs. At that gate, the formal SLO/exporter/retention/backup/Worker-progress closure was still pending. -- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress` for P2-06/P2-07. +- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress`; P2-06 is `pending_on_docs_ci`, and P2-07 is `not_started`. - Opened PR #1 for the trusted-local evaluation/Web credential/Run UX/SSE branch and obtained all four required GitHub Actions jobs for exact SHA `ab15862eab4870dda01fb079b44b509a7d737627` in run `33078921254`; this closes the previously documented no-PR/no-CI boundary for that SHA only. - Passed 205 backend non-infrastructure tests, five real PostgreSQL/Redis integration tests, 13 frontend tests, the isolated offline Mock smoke test, Ruff/ESLint/TypeScript checks, Vite production build, and SQLite/PostgreSQL Alembic upgrade/check/downgrade/upgrade gates. - Passed an isolated default-build Compose acceptance harness in all eight scenarios: topology/readiness, protocol-v1 baseline, API restart during execution, exact lease-owner SIGKILL and natural takeover, Redis stop/start with database reconciliation, pending cancellation, running cancellation plus duplicate delivery, and PostgreSQL `head -> 0001 -> head` protocol round-trip. diff --git a/README.md b/README.md index 5101883..c14d227 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ GitHub：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测工作台。它把模型注册、版本化 Benchmark、后台评测、逐题证据、汇总指标和排行榜放进一条可审计的本地流程，并以“默认离线、严格评分、结果可复现”为首要约束。 -当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选，并实现待仓库级收尾的 P2-06 可观测性/审计保留切片：PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger、typed audit 与 Worker progress 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 +当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选；P2-06 可观测性/审计保留切片处于 `pending_on_docs_ci`：实现及其精确 SHA 门禁已完成，仅剩本次 evidence closeout 文档 commit 自身的精确 SHA CI。PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger、typed audit 与 Worker progress 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 ## 当前状态 @@ -17,7 +17,9 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - Phase 2 候选已通过真实 PostgreSQL/Redis 和进程故障验证：除租约、心跳、fencing、幂等 Response 与数据库恢复外，Web/API managed Run 还具有 global/provider/model/run 四层数据库 admission、fixed-minute RPM/TPM、lifetime request/Token/cost budget、有限 backlog、公平 slice、逐 attempt reservation/settlement、typed audit 和历史延迟。 - 可信本地 CLI 已提供 MMLU-Pro 与 GPQA-Diamond 的固定来源转换、真实 OpenAI-compatible 预检、可恢复执行和完整报告导出；这是 Phase 3 的客观题垂直切片，不代表 Phase 2 或 Phase 3 已完成。 - 自动化、CI、Compose 故障验收和容量演练的模型执行都只使用 Mock；根据层级使用临时 SQLite 或隔离 PostgreSQL 16/Redis 7，不访问真实模型服务，也不产生模型费用。 -- Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。P2-06 已在当前工作树实现 `0005`/13 表 importer、Worker DB-time progress、固定低基数 Prometheus exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理；当前 lint、全量 test（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke、真实 PostgreSQL 16/Redis 7 integration（`33 passed, 0 skipped`）、隔离 migration、frontend build、Compose config、Prometheus `v3.5.0` 的八规则 `promtool` 校验，以及 dirty 工作树 9/9 Compose acceptance 与最新 capacity 均已通过。此前 code review 为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。独立 commit/push、clean-SHA Compose 证据和该精确 SHA 的远程 CI 尚未完成，因此 P2-06 仓库级收尾仍为 `in_progress`。P2-07 的数据库/keyring 配对 backup/restore、Redis 重建和剩余故障矩阵仍未完成。上述资格只限固定 Mock-only 单机控制面，不是生产或真实 Provider SLA；Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 +- Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +- P2-06 当前状态为 `pending_on_docs_ci`，不能标记为 `completed`。实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已 push 到当前分支并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。绑定同一 clean SHA 的 Compose acceptance 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`），Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0` 且 cleanup C/V/N 全空；clean capacity evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`），1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。这是 Mock-only 单机观测，不是生产或真实 Provider SLO；此前 dirty evidence 继续保留为历史，不替代该 clean-SHA 结果。实现门禁已全部完成，仅剩本 evidence closeout 文档 commit 的 push 与精确 SHA CI。 +- P2-07 明确为 `not_started`；数据库/keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵仍未开始，因此 Phase 2 不能完成。Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 最新、可复核的完成状态与测试证据以 [`docs/PROJECT_STATUS.md`](docs/PROJECT_STATUS.md) 和 [`docs/worklogs/`](docs/worklogs/) 为准；Roadmap 中的计划能力不等于已交付能力。 @@ -394,7 +396,7 @@ uv run llmbenchlab-audit-retention delete \ | --- | --- | --- | | Phase 0 | 项目治理、需求、架构、协议 | 已完成 | | Phase 1 | FastAPI + React + SQLite 的 MVP 垂直链路 | 已完成 | -| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 的 Mock-only 单机 v2 资格已闭环；P2-06 已实现且 dirty capacity/9/9 acceptance、修复后 staged 技术/安全终审已通过，尚待 commit/clean-SHA Compose/push/精确 SHA CI；P2-07 backup/restore 和剩余恢复矩阵未完成 | +| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 已闭环；P2-06 为 `pending_on_docs_ci`，实现 SHA `9a20676…` 已 push、PR #3 与实现 run `33164609388` 4/4、clean capacity/9/9 acceptance 全绿，仅剩 evidence-doc commit 自身精确 CI；P2-07 为 `not_started` | | Phase 3 | 合规标准 Benchmark 与隔离代码评测 | MMLU-Pro/GPQA 客观数据切片已交付；IFEval、沙箱及其余验收未完成 | | Phase 4 | LLM Judge、人工校准与 Arena | 计划中 | | Phase 5 | Agent、工具调用与 Live Benchmark | 计划中 | @@ -431,7 +433,7 @@ uv run llmbenchlab-audit-retention delete \ - Dataset Hash 用于一致性检查，不是发布者签名，也不能证明数据没有污染。 - `/tasks/metrics`、retained `/tasks/history`、单 Run typed audit、固定 Prometheus exporter、八条规则和显式 audit retention CLI 已实现；仓库仍不部署 Prometheus/Alertmanager、告警发送器或 trace，也不提供 WORM/数据库管理员防篡改、自动请求链路清理或对象存储上传。`resume` 的新 canary 仍不会追加独立事件。 - 逐题 Provider request ID、returned model、system fingerprint、finish reason 和 HTTP attempt count 已按字符/长度/秘密规则 fail closed 持久化并可导出；它们是关联证据，不是供应商真实性或账单证明。 -- 已保留 dirty-worktree 增强前基线、clean `665244e…` 容量候选和永久 `unqualified` 的 v1 资格历史；clean `b6a35fe…` 的 v2 `qualified` 结果也只证明固定 Mock 配置下的单机本地控制面，不是真实 Provider、生产 SLO/SLA、HA 或灾难恢复证明。仍无 SBOM 和生产部署支持。 +- 已保留 dirty-worktree 增强前基线、P2-06 dirty acceptance/capacity 历史、clean `665244e…` 容量候选和永久 `unqualified` 的 v1 资格历史；clean `b6a35fe…` 的 v2 `qualified` 与 clean `9a20676…` 的 P2-06 acceptance/capacity 结果都只证明各自固定 Mock 配置下的单机本地控制面，不是真实 Provider、生产 SLO/SLA、HA 或灾难恢复证明。仍无 SBOM 和生产部署支持。 ## 贡献 diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 0fd428b..5e5978c 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -1,6 +1,6 @@ -# 下一任务：完成 P2-06 仓库门禁，再推进 P2-07 恢复闭环 +# 下一任务：完成 P2-06 证据文档 CI，再启动 P2-07 -> 状态：`in_progress`；P2-06 功能、主要本地门禁、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已完成，独立 commit/clean-SHA Compose/push 与精确 SHA CI 尚未完成 +> 状态：`pending_on_docs_ci`；P2-06 实现、clean-SHA capacity/9/9 acceptance、push 与 implementation SHA CI 已完成，只待本次证据文档提交自身的精确 SHA CI > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) > 当前计划：[Phase 2 可观测性与审计保留](plans/2026-08-28-phase-2-observability-retention.md) > 当前日志：[2026-08-28 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) @@ -8,7 +8,7 @@ ## 当前事实 -P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，clean SHA 上的 `P2-local-control-plane-v2` 完成 1 warm-up + 5 measured、23/23 SLO，aggregate SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。这些结果只属于对应历史 SHA，不能替代当前 P2-06 工作树门禁。 +P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，clean SHA 上的 `P2-local-control-plane-v2` 完成 1 warm-up + 5 measured、23/23 SLO，aggregate SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。这些结果只属于对应历史 SHA，不能替代当前 P2-06 证据文档门禁。 当前 P2-06 已按 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 实现： @@ -19,55 +19,19 @@ P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 提供 canonical JSONL v1、严格文件/权限/大小/schema/hash/rollup 校验、真正离线 verify、精确 digest 绑定、默认不删除和 commit-outcome 分类。它不提供签名/WORM，也不替代整库+keyring backup。 - 全生产日志调用受字面量/无格式参数静态门禁；第三方动态消息固定化、结构化数值有限化，raw Uvicorn access handler 关闭。 -当前验证事实：合并定向套件全绿，并包含 importer committed-target integrity、retention no-op postverify、外部 logger extra、FIFO/line cap 与 PostgreSQL advisory/row-lock 终审回归；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 后 integration 为 `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、frontend production build、`docker compose config --quiet` 与 `prom/prometheus:v3.5.0` 的八规则 `promtool check rules` 通过。Dirty Compose acceptance 已 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`（SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`），Worker gauges `2/2/2/0/0`，`0005` populated refusal、isolated populated `0004` refusal、两层空库往返和 cleanup C/V/N empty 均成功。最新 dirty capacity 也通过，evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`（SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`）：18 Runs/270 Responses/270 question executions/271 reservations/1229 audit，0 error/drift/duplicate/PEL/lag，Worker expected 2 且 cleanup C/V/N/image 全空；这是 offline Mock 非 SLO 观察值。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。最新 hydration/import integrity 目标集 `67 passed`。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后未擅自迁移。当前仍无 P2-06 独立 commit、clean-SHA Compose、远程 SHA 或 CI 链接；不得把 P2-06 写成 `completed`。 +当前验证事实：合并定向套件全绿；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 integration 为 `33 passed, 0 skipped`，隔离 migration、frontend build、Compose config 与 Prometheus v3.5.0 八规则校验均通过。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）在 implementation SHA `9a20676dcf545040782f04c166205d0043345753` 上 `dirty=false`、9/9，Worker `2/2/2/0/0`、两级 populated refusal、两层空库往返与 cleanup C/V/N empty。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）同样 `dirty=false`：1W/2W/burst `7.267474/12.962228/9.333604 q/s`，18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag；expected Worker=2、stalled/shortfall=0，故障恢复后的瞬时 registered/live=3，cleanup C/V/N/image=0。该结果是 offline Mock 非 SLO。structured-extra High 与 Worker `__main__` logger Medium 均已修复，最终技术/安全复核为 0 Blocker/High/Medium。实现提交已 push，PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 精确绑定该 SHA并4/4 success。默认用户 SQLite 尚未在 head且未擅自迁移；证据文档 SHA CI 绿色前 P2-06 仍不能写成 `completed`。 -## 立即执行：P2-06 仓库级收尾 +## 立即执行：P2-06 证据文档收尾 -### 1. 完成实现终审 - -- 复核 Worker registration/stop/late-flush、scan/claim/progress/heartbeat event 定义，确认 timer 无事件时零写入、stale 边界 `last_seen_at == cutoff` 为 live。 -- 复核 exporter cancellation/single-flight：请求取消后 DB 线程终止前不得提前释放 gate；audit 第 `50,001` 行必须整次 `503`，不能截断；数据库/audit/renderer 错误不得返回部分 exposition。 -- 复核 archive 从同一已打开 descriptor 完成 verify/delete/restore，`verify` fresh process 在无效 DB 配置下仍不导入/创建 engine；delete/restore 只处理 archive 列出的精确完整事实，不能做宽 cutoff 删除。 -- 复核 `0005`、prepare fingerprint、13 表 importer、Compose expected Worker count 与 acceptance/capacity 两 Worker 拓扑一致；终审确认 importer committed-target canonical integrity、retention no-op postverify 和 PostgreSQL advisory/row-lock 门禁继续生效。 -- 检查所有生产 logger 调用、第三方 handler、Uvicorn access 和固定错误，确认不反射异常、DSN、路径或 argv marker。 - -### 2. 完成剩余验证 - -以下门禁已经执行并记录，无后续代码变化时不需要为碰绿重复运行： - -```bash -make lint -make test -make smoke -(cd backend && uv run pytest -m integration) -(cd frontend && npm run build) -docker compose config --quiet -``` - -迁移验证使用隔离临时 SQLite 和 PostgreSQL；不要为文档收尾擅自 upgrade 默认用户 SQLite。当前剩余命令/检查是： - -```bash -make phase2-capacity -# dirty acceptance 已通过；commit 后仍须在 clean SHA 重跑两项 -make phase2-acceptance -git diff --check -``` - -- 自动化只能使用 Mock/MockTransport/stub，不调用真实或付费 Provider。 -- `prom/prometheus:v3.5.0` 内的 `promtool check rules` 已验证八条规则成功；最终 diff 只需确认规则文件未再变化，若变化则必须重跑。 -- 已完成的真实 PostgreSQL 路径覆盖 `0005` upgrade/check/空库往返、audit retention advisory/row lock 往返和 13 表 importer；dirty acceptance 已证明 application populated `0005 -> 0004` 拒绝、isolated populated `0004 -> 0003` 拒绝及两层空库往返。clean SHA 仍须重跑并绑定 evidence。 -- 检查 Markdown 内部链接、固定八条 Runbook anchor、exposition metric 名、规则引用、文档版本和实际实现一致。 - -### 3. Commit、push 与精确 SHA CI - -- 复核完整 diff/staged diff，排除真实 Key、Authorization/Cookie、credential DSN、ciphertext/nonce/keyring、archive 产物、调试输出和无关生成物。 -- 修复后 staged 技术/安全终审和最新 dirty capacity 已通过；形成一个独立、可审查的 P2-06 commit，在该 clean SHA 上重跑 capacity/acceptance 并绑定 evidence，再普通 push 到 `origin/codex/complete-evaluation-workflow`，继续通过 PR #2 触发 CI；禁止 force push。 -- 等待该**精确 commit SHA** 的四个必需 GitHub Actions job 全部成功。任何失败都读取日志、修复、形成新 commit/push，再等待新 SHA；历史绿色或仅本地通过不能替代。 -- 绿色后再把 P2-06 状态改为 `completed`，并补齐 commit、branch、Actions URL、命令、测试数、跳过/未运行项、Compose cleanup、secret scan 与 `git status --short` 证据。Phase 2 仍因 P2-07 保持 `in_progress`。 +1. 复核本次证据文档 diff，确保 clean 与历史 dirty 结果不混用；capacity 的 `1 failed attempt` / `1 queue-notification error` 是注入故障的预期对账，不得笼统写成零，acceptance 也不得误写成清理了 build image。 +2. 检查 Markdown 链接、`git diff --check`、staged diff、秘密/凭据/raw evidence/本机路径/生成物；只提交文档，不修改实现或 evidence artifact。 +3. 形成独立 evidence-doc commit，普通 push 当前工作分支，通过 PR #3 触发 CI。 +4. 等待该**文档提交精确 SHA** 的四个必需 GitHub Actions job 全部 success。失败时读取日志、修复并对新 SHA 重跑；不得用实现 SHA 的绿色替代。 +5. 文档 SHA 绿色后，另做状态收尾提交：把 P2-06 标为 `completed`、计划步骤 6 标完成、NEXT_TASK 转为 P2-07；Phase 2 继续 `in_progress`。状态提交也须 push 并等待自身精确 SHA CI。 ## P2-07：下一独立切片 -P2-06 精确 SHA 全绿后，新建独立计划、工作日志和必要 ADR，完成： +P2-07 当前为 `not_started`。P2-06 文档门禁全绿后，新建独立计划、工作日志和必要 ADR，完成： - PostgreSQL backup → 空目标 restore → Alembic `20260828_0005` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 - 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 @@ -84,12 +48,12 @@ P2-06 精确 SHA 全绿后，新建独立计划、工作日志和必要 ADR， ## Definition of Done -- P2-06：功能、文档、迁移与测试合同一致；全量本地/真实基础设施门禁通过；独立 commit 已 push；该精确 SHA 四个必需 CI job 全绿；状态/计划/日志记录完整且无秘密或生成归档。 +- P2-06：功能、文档、迁移与测试合同一致；全量本地/真实基础设施门禁通过；实现与证据文档各有独立 commit 并 push；对应精确 SHA 的四个必需 CI job 均全绿；状态/计划/日志记录完整且无秘密或生成归档。 - Phase 2：只有 P2-06 上述门禁和 P2-07 backup/restore、Redis 重建、告警处置与完整故障矩阵均有可复核证据后，才可评估 `completed`。 -- 任一关键项未运行、失败或未形成精确 SHA 远程绿色时，相关任务和 Phase 2 必须保持 `in_progress`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展、WORM 或 Provider exactly-once。 +- 任一关键项未运行、失败或未形成精确 SHA 远程绿色时，相关任务不得标记 `completed`；当前 P2-06 保持 `pending_on_docs_ci`，Phase 2 保持 `in_progress`，且不得宣称生产 HA、灾难恢复 SLA、无限横向扩展、WORM 或 Provider exactly-once。 ## 可直接复制给 Codex 的任务指令 ```text -继续执行 docs/NEXT_TASK.md。当前 P2-06 功能以及 lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 和修复后 staged 技术/安全终审已通过，不要重做 P2-01，也不要跳到 P2-07。现在形成独立 commit，在 clean SHA 重跑 capacity/acceptance，普通 push 当前工作分支并等待该精确 SHA 的四个 CI job 全绿；失败就修复并对新 SHA 重跑。P2-06 绿色后再独立推进 PostgreSQL+keyring backup/restore、Redis 重建、告警响应和剩余恢复矩阵。自动化只用 Mock/Stub；Phase 2 在 P2-07 完成前保持 in_progress。 +继续执行 docs/NEXT_TASK.md。P2-06 implementation SHA 9a20676dcf545040782f04c166205d0043345753 的 clean capacity/9/9 acceptance 与 GitHub Actions run 33164609388 4/4 已通过；不要重跑碰绿，也不要提前启动 P2-07。现在复核并提交 clean evidence 文档，普通 push 当前工作分支并等待该文档精确 SHA 的四个 CI job 全绿。绿色后再做状态收尾提交，把 P2-06 标 completed、NEXT_TASK 转入新建 P2-07 计划；Phase 2 仍保持 in_progress。自动化只用 Mock/Stub，禁止公开 raw evidence 或真实凭据。 ``` diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index e49c0bc..ec33cd6 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -6,7 +6,7 @@ - Phase 0 — 项目治理和架构：`completed`（2026-08-24） - Phase 1 — MVP 垂直链路：`completed`（2026-08-25） -- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计与 P2-01 单机资格已完整交付；P2-06 已实现但仓库级收尾仍进行中，P2-07 运维恢复仍有缺口） +- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计与 P2-01 单机资格已完整交付；P2-06 为 `pending_on_docs_ci`，P2-07 为 `not_started`） - Phase 3 — 标准 Benchmark 与代码评测：`in_progress`（仅可信本地 MMLU-Pro/GPQA-Diamond 客观题提前切片） - Phase 4–6：`planned` @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`，PR [#2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)。P2-01 v2 实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 已普通 push，GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 的四个必需 job 均成功；证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 也已普通 push，其精确 SHA 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 4/4 成功。后者完成 P2-01 仓库级收尾，不改变性能证据绑定的实现 SHA。P2-06 当前改动尚未形成独立 commit/push，也没有可引用的精确 SHA CI；不得把上述历史远程门禁归因于当前工作树。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。P2-06 当前为 `pending_on_docs_ci`，只剩本次 evidence closeout 文档 commit 的 push 与精确 SHA CI，不能提前标记 `completed`。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -41,19 +41,19 @@ - 精确 SHA `665244e…` 的增强 capacity 使用有限 policy、PostgreSQL 16、Redis 7 与两个 Worker 完成：并发 backlog 精确为 4 个 `202` + 2 个 typed `429`，cooperative yield 与跨 Model 公平顺序均有 durable audit 证据，最终 18 Runs/270 Responses/271 ledger/1229 audit 对账且无 active/reserved/overdrawn 漂移。 - P2-01 的 `P2-local-control-plane-v2` 已在干净 SHA `b6a35fe…` 从零执行 1 次 warm-up + 恰好 5 次 measured trial，本次 invocation 的 `discarded_trials=0`；四个 cell、23/23 SLO 与每轮 hard invariant 全部通过，容量模型为 `qualified`。aggregate SHA-256 为 `a76d167b…d0d9`，六轮均精确完成 22 Runs/330 Responses/330 QuestionExecutions/331 reservations，并清理本项目容器、卷、网络和唯一 build image。历史 v1 aggregate `f993c11f…e3b2` 继续保持 `failed/not_qualified`。 -## 当前 P2-06 实现（仓库级收尾进行中） +## P2-06 实现（`pending_on_docs_ci`） -- [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；Alembic head 已在当前工作树扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 +- [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；实现 SHA `9a20676dcf545040782f04c166205d0043345753` 将 Alembic head 扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 - `GET /api/v1/metrics/prometheus` 已实现固定 Prometheus text `0.0.4` gauge：一个 DB-time 读快照、15 分钟 typed-audit 窗口、1 小时 Run latency、硬读取上限、固定 enum label、整次 fail-closed 与每 API 进程 single-flight。`deploy/observability/` 提供固定八条规则和安全抓取示例；仓库不部署 Prometheus、Alertmanager 或通知发送器。 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 已实现 canonical JSONL v1、严格权限/大小/行/schema/hash/rollup 校验、离线 verify、精确 digest 绑定、默认不删除、双方言事务与 commit outcome 分类。Archive 是敏感运维文件，hash 只用于完整性/绑定，不是签名或 WORM，也不替代 P2-07 的数据库+keyring 备份。 - importer 当前合同为 `0005` 的 13 表精确 count/PK/content digest；live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 - 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 -- 上述实现的合并定向套件已全绿；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck），`make test` 为后端 `916 passed, 33 skipped`、前端 `38 passed`，`make smoke` 为 `1 passed, 7 deselected`，临时 PostgreSQL 16/Redis 7 migration/check 和真实 integration 为 `33 passed, 0 skipped`，临时 SQLite head→0001→head/check、frontend production build、`docker compose config --quiet` 与临时 `prom/prometheus:v3.5.0` 的八规则 `promtool check rules` 均通过。Dirty 工作树 Compose acceptance 已 9/9 通过，Worker gauges 为 `2/2/2/0/0`，两级 populated downgrade refusal、两层空库往返和 cleanup 空检查均成功；最新 dirty capacity 也通过，最终 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit，0 error/drift/duplicate/PEL/lag 且 cleanup 全空。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。最新 hydration/import integrity 目标集 `67 passed`。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。独立 commit、clean-SHA Compose、普通 push 与精确 SHA 四个必需 CI job 仍待完成。 +- 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 ## 仍未完成 -- P2-06：实现与 lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 及修复后 staged 技术/安全终审已通过，但独立 commit、clean-SHA Compose、push 与精确 SHA 远程 CI 尚未完成；在这些门禁绿色前保持 `in_progress`。 -- P2-07：PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练。P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 +- P2-06：状态为 `pending_on_docs_ci`，实现、clean-SHA Compose evidence、push 和实现精确 SHA CI 全部完成；只剩本 evidence closeout 文档 commit 的 push 与精确 SHA CI，因此仍不能标记 `completed`。 +- P2-07：状态为 `not_started`。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 @@ -73,24 +73,27 @@ | 验证 | 实际结果 | 当前结论 | | --- | --- | --- | -| P2-06 合并定向套件 | 全绿；不在最终全量前记录或推断全量测试数 | 目标实现回归通过，不替代完整 `make test` | -| P2-06 `make lint` | Ruff 152 files、format check、ESLint、TypeScript typecheck 全绿 | 当前未提交工作树通过 | -| P2-06 `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed` | 当前未提交工作树通过；只用 Mock/Stub | +| P2-06 合并定向套件 | 全绿；随后完整 `make test` 也已通过 | 目标实现回归与全量门禁均通过 | +| P2-06 `make lint` | Ruff 152 files、format check、ESLint、TypeScript typecheck 全绿 | 实现 SHA `9a20676…` 的本地门禁通过 | +| P2-06 `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed` | 实现 SHA `9a20676…` 的本地门禁通过；只用 Mock/Stub | | P2-06 Mock smoke | `1 passed, 7 deselected` | 完全离线通过 | -| P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；尚未绑定远程 SHA | +| P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；实现 SHA 的远程 integration job 也已成功 | | P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；默认用户 SQLite 非 head 的 check 失败后未迁移 | current head `20260828_0005`；用户数据保持原状 | | P2-06 build/config | frontend build 成功（保留 662.39 kB chunk warning）；Compose config exit 0 | 从根目录误跑 npm 的失败已记录并用正确目录重跑通过 | | P2-06 Prometheus 规则 | `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | 临时容器验证；仓库仍不部署 Prometheus/Alertmanager | | P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`、`0005`/isolated `0004` populated refusal、两层空库往返、cleanup C/V/N empty | -| P2-06 dirty capacity | 最新 artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453` 通过 | `dirty=true`；18/270/270/271/1229，0 error/drift/duplicate/PEL/lag，Worker expected=2、cleanup C/V/N/image=0；offline Mock、非 SLO | +| P2-06 dirty capacity | 历史 artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453` 通过 | `dirty=true`；18/270/270/271/1229，0 error/drift/duplicate/PEL/lag，Worker expected=2、cleanup C/V/N/image=0；offline Mock、非 SLO | +| P2-06 clean acceptance | 9/9；artifact `llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，`dirty=false`，commit `9a20676…` | Worker `2/2/2/0/0`；cleanup C/V/N empty | +| P2-06 clean capacity | artifact `llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`，`dirty=false`，commit `9a20676…` | QPS `7.267474/12.962228/9.333604`，wall `8.255963/4.628834/6.428385s`；18/270/270/271/1230；0 question error/drift/duplicate/PEL/lag；expected=2、shortfall=0；cleanup C/V/N/image=0、image `1/1/0/0`；Mock、非 SLO | | P2-06 补充静态检查 | 过宽 `scripts/` Ruff 命令暴露 93 条既有 modernization 告警；`--select E,F,I` 通过 | 如实保留首次结果，不把范围外历史告警归为本次回归 | -| P2-06 staged 技术/安全终审 | structured-extra High 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 当前 dirty 工作树终审通过 | -| P2-06 剩余门禁 | commit、clean-SHA Compose、push、精确 SHA CI 待完成 | 不得标记仓库级收尾完成 | -| P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 仍 open，未合并 | +| P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入实现 SHA `9a20676…` | +| P2-06 实现远程 CI | PR #3；`9a20676dcf545040782f04c166205d0043345753` 的 run `33164609388` 4/4 | 实现精确 SHA 门禁完成 | +| P2-06 剩余门禁 | 本 evidence closeout 文档 commit 的 push 与精确 SHA CI | 当前 `pending_on_docs_ci`；不得标记 `completed` | +| P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | -| 最新本地 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现冻结树通过 | -| 最新真实 PostgreSQL/Redis integration | `29/29 passed` | 本地通过；同一实现 SHA 的远程 integration 亦成功 | +| P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | +| P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | | 最新本地 `make smoke` | `1 passed, 7 deselected`，仅 Mock | 本地冻结树通过；没有调用真实 Provider | | 定向治理/API/Worker | 目标套件零失败；早期独立审计 `218 passed`；完整性边界集合 `18 passed` | 已被最终全量、真实 integration 与精确候选 evidence 补充 | | SQLite/PostgreSQL migration | 隔离 SQLite 与真实 PostgreSQL prepare/upgrade/downgrade guard/upgrade/check 通过 | 候选与远程 integration 覆盖 | @@ -114,4 +117,4 @@ ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 是当前合同：先完成当前 P2-06 工作树的独立 commit、clean-SHA Compose、push、精确 SHA CI 与证据收尾，再单独推进 P2-07 的数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 当前仅剩 evidence closeout 文档 commit 的 push 与精确 SHA CI；其通过前状态保持 `pending_on_docs_ci`。之后才可单独启动仍为 `not_started` 的 P2-07 数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 68309a0..3aa2e05 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,7 +30,7 @@ | --- | --- | --- | --- | --- | | Phase 0 | 项目治理和架构 | 可执行的需求、架构、协议、ADR 与持续文档流程 | `completed` | [PHASE-0-GOVERNANCE.md](phases/PHASE-0-GOVERNANCE.md) | | Phase 1 | MVP 垂直链路 | Mock 模型到 Run、逐题结果与排行榜的离线闭环 | `completed` | [PHASE-1-MVP.md](phases/PHASE-1-MVP.md) | -| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计与 P2-01 已交付；P2-06 已实现待仓库级收尾，P2-07 恢复闭环待完成 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | +| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计与 P2-01 已交付；P2-06 为 `pending_on_docs_ci`，P2-07 为 `not_started` | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | | Phase 3 | 标准 Benchmark 与代码评测 | 已有 MMLU-Pro/GPQA 可信本地切片；IFEval、沙箱与完整插件体系待完成 | `in_progress` | [PHASE-3-BENCHMARKS.md](phases/PHASE-3-BENCHMARKS.md) | | Phase 4 | Judge、Arena 与长上下文 | 可校准 Judge、Pairwise Judge、个人 Arena 和长上下文评测 | `planned` | [PHASE-4-JUDGE-ARENA.md](phases/PHASE-4-JUDGE-ARENA.md) | | Phase 5 | Agent、私有与 Live Benchmark | 工具调用轨迹、隔离私有集和持续更新的 Live Benchmark | `planned` | [PHASE-5-AGENT-LIVE.md](phases/PHASE-5-AGENT-LIVE.md) | @@ -183,7 +183,7 @@ - managed Run 冻结 policy/hash 与显式 overrides；global/provider/model/run 四层 concurrency、固定窗口 RPM/TPM、global/run lifetime request/Token/USD budget 和逐 Provider HTTP attempt ledger。 - materialized counter 只作 ledger 投影；counter、policy/hash 或 Run override 漂移 fail closed。confirmed pre-send release 按 ADR-0011 不消耗未发送 HTTP retry。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model 公平排序。 -- typed audit、Run audit、task history/latency、Provider metadata、credential 非秘密事件和前端治理状态；当前工作树另已实现固定低基数 Prometheus exporter/八条规则、canonical retention archive/verify/reconcile/restore/delete、Worker DB-time progress/liveness 聚合与全日志源治理。 +- typed audit、Run audit、task history/latency、Provider metadata、credential 非秘密事件和前端治理状态；P2-06 实现 SHA `9a20676…` 另交付固定低基数 Prometheus exporter/八条规则、canonical retention archive/verify/reconcile/restore/delete、Worker DB-time progress/liveness 聚合与全日志源治理。 - enhanced Mock-only capacity/acceptance、真实 PostgreSQL 竞争测试，以及绑定 clean SHA 的正式多轮单机控制面资格；backup/restore 仍待完成。 ### 非目标 @@ -202,12 +202,12 @@ | ID | 状态 | 已交付 / 剩余范围 | | --- | --- | --- | | P2-01 一致性与容量设计 | 已交付 | ADR-0012～0014、DB truth/lease/fencing/治理、v2 四 cell 多轮统计、恢复与连接模型已实现；clean SHA `b6a35fe…` 的 1+5 资格为 23/23、`qualified`；证据文档 `875f13a…` 的精确 SHA CI 4/4 成功 | -| P2-02 PostgreSQL 迁移 | 切片已实现 | `0002`/`0003` 可靠性与凭据基础、`0004` 六类治理/审计表已通过既有门禁；当前 P2-06 工作树增加 `0005`、13 表 importer、live-Worker preflight 与 downgrade guard，尚待本切片全门禁/精确 SHA CI | -| P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付；当前 P2-06 工作树增加 generation 级 DB-time progress，dependency probe 仍只检查 capability | +| P2-02 PostgreSQL 迁移 | 切片已实现 | `0002`/`0003` 可靠性与凭据基础、`0004` 六类治理/审计表已通过既有门禁；P2-06 实现 SHA `9a20676…` 增加 `0005`、13 表 importer、live-Worker preflight 与 downgrade guard，实现精确 SHA CI 已通过 | +| P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付；P2-06 实现 SHA `9a20676…` 增加 generation 级 DB-time progress，dependency probe 仍只检查 capability | | P2-04 生命周期可靠性 | 可靠基础已交付 | retry/取消/恢复/dead-letter/终态重算及三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；外部调用仍不保证 exactly-once | | P2-05 并发治理 | 切片已实现 | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序和完整性 fail-closed 已实现；真实 PG/capacity/acceptance/精确 SHA CI 候选门禁已通过 | -| P2-06 可观测性 | `in_progress` | typed audit/history、Run latency、Provider/credential evidence 基线已交付；exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现，lint/test/integration/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已通过；commit/clean-SHA Compose/push 和精确 SHA CI 未完成 | -| P2-07 验证与运维 | 切片已实现 | enhanced capacity/PG tests、正式 v2 单机资格、精确候选 evidence 和运维/性能文档已交付；backup/restore、告警响应与完整失败矩阵未完成 | +| P2-06 可观测性 | `pending_on_docs_ci` | exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现；`9a20676…` 已 push、PR #3、实现 run `33164609388` 4/4，clean capacity/9/9 acceptance 全绿；仅剩 evidence-doc commit 自身的精确 SHA CI，不能标记 `completed` | +| P2-07 验证与运维 | `not_started` | P2-01/P2-06 已提供 capacity/PG/acceptance 基线，但 P2-07 本身的数据库+keyring backup/restore、Redis 重建、告警响应与完整失败矩阵尚未开始 | ### 验收标准 @@ -215,11 +215,12 @@ - `delivered`：真实 PostgreSQL 并发领取只有一个有效 lease；自然过期后由递增 fencing token 接管，陈旧 owner 写入被拒绝。 - `delivered/partial`：pending/running 取消有真实 Compose 证据；有限重试、超时和 dead-letter 有自动化状态机/Runner 证据，但尚未把所有失败组合都纳入完整生产式故障演练。 - `delivered`：历史 `0004` / 12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 -- `implemented/pending_gate`：当前工作树的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已实现；后端 `916 passed, 33 skipped`、前端 `38 passed`、真实 PG16/Redis7 `33 passed, 0 skipped`、八规则 promtool 与 dirty capacity/9/9 acceptance 已通过。此前代码终审为 0 Blocker/High/Medium；随后 staged 安全审查发现的 structured-extra 反射 High 与 `python -m app.worker` logger Medium 均已修复，最新 76-file staged 技术/安全复核重新收敛为 0 Blocker/High/Medium。独立 commit、clean-SHA Compose、push 和该精确 SHA 的 CI 尚未完成，因此 P2-06 未完成仓库级收尾。 +- `pending_on_docs_ci`：P2-06 的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已在 SHA `9a20676dcf545040782f04c166205d0043345753` 实现；本地 lint/test/integration/rules 与修复后 76-file 技术/安全终审全绿，实现已 push 到 PR #3，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）9/9、Worker `2/2/2/0/0`、cleanup C/V/N empty；clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）记录 QPS `7.267474/12.962228/9.333604`、wall `8.255963/4.628834/6.428385s`、18/270/270/271/1230、0 question error/drift/duplicate/PEL/lag、expected=2/shortfall=0 与 cleanup C/V/N/image=0、image `1/1/0/0`。两者均为 clean-SHA Mock-only evidence，不是 SLO；此前 dirty evidence 保留为历史。只剩本 evidence-doc commit 的精确 SHA CI，因此 P2-06 尚不能 `completed`。 - `delivered`：`P2-local-control-plane-v2` 在 clean SHA `b6a35fe…` 完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 和 `qualified` 容量模型；只限定记录的 Mock-only 单机拓扑。 -- `not_met`：backup/restore 仍未完成；三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 +- `not_started`：P2-07 backup/restore、Redis 重建、告警响应与完整恢复矩阵尚未开始；三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 - `delivered`：治理实现 SHA `665244e…` 已 push；本地 enhanced capacity、9/9 acceptance 与远程 run `33099260233` 4/4 均通过。 - `delivered`：P2-01 v2 实现 SHA `b6a35fe…` 已 push；本地 aggregate SHA-256 `a76d167b…d0d9` 为 23/23，实现 run `33146681285` 4/4 成功；证据文档 commit `875f13a…` 已 push，收尾 run `33150080341` 4/4 成功。 +- `delivered`：P2-06 实现 SHA `9a20676…` 已 push 至 PR #3，实现 run `33164609388` 4/4 成功，绑定同一 SHA 的 clean capacity 与 9/9 acceptance 已通过；当前仅 evidence closeout 文档 CI 待完成。 - `delivered`：既有 API、离线 Mock Smoke 和 `llmbenchlab-protocol-v1` 评分/聚合回归继续通过，没有调用真实 Provider。 ### 风险 @@ -234,13 +235,13 @@ - 已交付基础：PostgreSQL/SQLite migration、Redis/Worker/lease/fencing/recovery、六服务 Compose 与历史故障证据。 - 已交付候选切片：`0004`、历史 12 表 importer、四层治理/ledger/backpressure/fairness、typed audit/history/Provider/credential evidence、UI 状态、enhanced capacity/PG tests 与运维文档。 -- 已实现待门禁切片：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator 和全日志源治理。 +- 已完成实现门禁、待 evidence-doc CI 的切片：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator 和全日志源治理。 - 治理候选门禁已完成；P2-01 的 v2 多轮资格、实现 commit/push、实现 SHA 4/4 CI 与证据文档精确 SHA 4/4 CI 也已完成，P2-01 仓库级收尾完成。 -- 阶段正式剩余：先完成 P2-06 的 commit、clean-SHA capacity/acceptance、push、精确 SHA CI 与证据收尾；随后完成 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 +- 阶段正式剩余：先完成 P2-06 evidence closeout 文档 commit 的 push 与精确 SHA CI；随后再启动 `not_started` 的 P2-07 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 ### 状态 -`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 功能已在当前工作树实现，但仓库级门禁尚未闭环，P2-07 backup/restore 与完整恢复演练仍缺失。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 +`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 为 `pending_on_docs_ci`，实现与 clean evidence 门禁已完成但本 evidence-doc commit 尚无精确 SHA CI；P2-07 为 `not_started`。不得把 P2-06、Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 ## 6. Phase 3：标准 Benchmark 与代码评测 diff --git a/docs/TESTING.md b/docs/TESTING.md index 0787a57..84f1f40 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -27,7 +27,7 @@ LLMBenchLab 的默认验收路径必须完全离线、可重复且不产生模 | 构建检查 | 确认生产前端可编译打包 | `npm run build` | 安装完成后不需要 | | 配置检查 | 校验 Compose 插值和服务定义 | `docker compose config` | 不启动 Provider | -单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有已交付自动化；P2-06 当前又加入 Worker progress、exporter/规则、retention 与全日志源回归。合并定向、lint、全量 test、smoke、真实 integration、隔离 migration、build、Compose config、Prometheus 规则解析、dirty 工作树 capacity 与 9/9 acceptance 已全绿；structured-extra High 与 Worker `__main__` logger Medium 修复后，最新 76-file staged 技术/安全终审为 0 Blocker/High/Medium。当前仍待 clean-SHA Compose 证据与精确 SHA CI，不能把已通过的本地层级写成 P2-06 仓库级收尾。Phase 2 还缺 P2-07 backup/restore、剩余恢复矩阵、生产 SLA/HA 证明，因此继续 `in_progress`。 +单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有已交付自动化；P2-06 又加入 Worker progress、exporter/规则、retention 与全日志源回归。合并定向、lint、全量 test、smoke、真实 integration、隔离 migration、build、Compose config、Prometheus 规则解析与提交前 dirty capacity/9/9 acceptance 已全绿；structured-extra High 与 Worker `__main__` logger Medium 修复后，76-file staged 技术/安全终审为 0 Blocker/High/Medium。实现已在 clean commit `9a20676dcf545040782f04c166205d0043345753` 上重跑 capacity/acceptance 并绑定 `dirty=false` evidence，该精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 必需 job 成功。P2-06 仓库收尾仍为 `pending_on_docs_ci`；Phase 2 保持 `in_progress`，P2-07 保持 `not_started`。 ## 3. 环境准备 @@ -352,7 +352,7 @@ Smoke Test 证明 API 与 Worker 责任边界以及数据库驱动的最小离 真实模型验收应在隔离的本地数据库上先运行 `--limit`，人工核对模型发现、付费 canary、请求上界、逐题错误、Provider 账单和报告三文件，再决定是否 `--full`。这项手工操作不得写入自动化测试结果或 CI 通过数；如果本次没有真实 API URL/Key，就应明确记录“未运行”，不能用 MockTransport 结果替代“真实 Provider 已验证”。 -当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider metadata、Worker progress、固定 exporter/八规则和 canonical audit retention；合并定向套件、最新 dirty capacity 与 9/9 acceptance 全绿，真实 PostgreSQL/Redis integration 为 `33 passed, 0 skipped`。clean-SHA Compose 和精确 SHA CI 仍待，已知观测缺口还包括 `resume` canary 未追加独立 audit event、仓库不部署 sender/trace/WORM；不能把这些边界写成 Provider exactly-once、生产监控平台或 Phase 2 完成。 +当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider metadata、Worker progress、固定 exporter/八规则和 canonical audit retention；合并定向套件、提交前 dirty capacity/9/9 acceptance 与实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 clean capacity/9/9 acceptance 全绿，真实 PostgreSQL/Redis integration 为 `33 passed, 0 skipped`，同一实现 SHA 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。已知观测缺口还包括 `resume` canary 未追加独立 audit event、仓库不部署 sender/trace/WORM；不能把这些边界写成 Provider exactly-once、生产监控平台或 Phase 2 完成。P2-06 仓库收尾的唯一剩余门禁是证据文档 commit/push 及其精确 SHA CI，状态为 `pending_on_docs_ci`；P2-07 尚未开始。 ## 9. 前端测试要求 @@ -381,13 +381,13 @@ GitHub Actions 对 `main` push 和 Pull Request 触发四类 job： | Job | 必需检查 | 隔离与失败规则 | | --- | --- | --- | | `backend` | Ruff lint/format；临时 SQLite `upgrade -> 0001 -> head`/check；`pytest -m "not integration"` | 临时 SQLite；不启动 PostgreSQL/Redis；离线 Mock/MockTransport | -| `backend-integration` | 真实 PostgreSQL migration 往返；PostgreSQL/Redis/importer/audit-retention 的 31 个 `integration` 用例 | Actions service 容器；JUnit 必须收集非零用例且零 skip，否则 job 失败 | +| `backend-integration` | 真实 PostgreSQL migration 往返；PostgreSQL/Redis/importer/audit-retention 的 33 个 `integration` 用例 | Actions service 容器；JUnit 必须收集非零用例且零 skip，否则 job 失败 | | `full-stack-reliability` | `python3 scripts/phase2_acceptance.py` 的隔离 Compose 九场景 | 唯一项目/卷、随机 loopback 端口、Mock-only；总是上传已脱敏 evidence，脚本总是精确清理 | | `frontend` | ESLint、Vitest 组件测试、production build（`tsc -b` + Vite） | `npm ci` 锁定依赖；fetch/Recharts stub；`VITE_API_BASE_URL=/api/v1` | -CI 不配置 Provider Key、不调用真实模型，也不在线下载 MMLU-Pro/GPQA。PostgreSQL/Redis 是测试依赖，不是 Provider 网络；标准数据转换只使用 fixture fetcher。`P2-local-control-plane-v2` 的 validator、统计、四-cell 编排失败路径、ledger projection 和 exact-project cleanup 合同可以进入普通自动化，但 GitHub-hosted runner 不运行 `make phase2-slo` 的绝对吞吐/延迟门禁：共享 runner 的 CPU、内存和 Docker 调度不是稳定性能实验室。正式实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功；它是同一 SHA 的正确性门禁，不替代本机 1+5。所有必需 job 通过后才能合并；跳过用例、降低断言或使用 `continue-on-error` 都不算修复。具体分支和 Review 门槛见 [GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md)。 +CI 不配置 Provider Key、不调用真实模型，也不在线下载 MMLU-Pro/GPQA。PostgreSQL/Redis 是测试依赖，不是 Provider 网络；标准数据转换只使用 fixture fetcher。`P2-local-control-plane-v2` 的 validator、统计、四-cell 编排失败路径、ledger projection 和 exact-project cleanup 合同可以进入普通自动化，但 GitHub-hosted runner 不运行 `make phase2-slo` 的绝对吞吐/延迟门禁：共享 runner 的 CPU、内存和 Docker 调度不是稳定性能实验室。P2-01 正式实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功；它是同一 SHA 的正确性门禁，不替代本机 1+5。P2-06 实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 也已 4/4 成功。所有必需 job 通过后才能合并；跳过用例、降低断言或使用 `continue-on-error` 都不算修复。具体分支和 Review 门槛见 [GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md)。 -P2-06 当前工作树的阶段性证据如下；尚未完成项必须保持 pending，不能沿用任何历史 SHA 的绿色结果： +P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史过程记录，仓库收尾以 `9a20676dcf545040782f04c166205d0043345753` 绑定的 clean evidence 和精确 SHA CI 为准；证据文档自身的 commit/push/CI 未完成前，状态必须保持 `pending_on_docs_ci`： | 验证 | 当前实际结果 | | --- | --- | @@ -401,11 +401,15 @@ P2-06 当前工作树的阶段性证据如下；尚未完成项必须保持 pend | Compose config | `docker compose config --quiet` exit 0 | | Prometheus rules | 临时 `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | | Ruff scripts 补充检查 | 对 `scripts/` 的过宽默认 Ruff 命令报告 93 条既有 modernization 告警；按本任务合同使用 `--select E,F,I` 后通过，未把既有告警冒充本次失败 | -| Dirty Compose acceptance | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和容器/卷/网络空清理均通过 | -| Dirty Compose capacity | 最新脚本通过；evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`，最终 18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1229 audit，0 error/drift/duplicate/PEL/lag，expected Worker=2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | -| Staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 修复目标集 `67 passed` | -| 仍待门禁 | implementation commit、clean-SHA Compose、push 与精确 SHA CI 尚待完成 | -| Clean-SHA Compose / remote | 尚无 P2-06 独立 commit、clean-SHA capacity/acceptance、push、精确 SHA 或 Actions URL；P2-06 仓库级收尾保持 `in_progress` | +| 提交前 dirty Compose acceptance（历史） | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和容器/卷/网络空清理均通过 | +| 提交前 dirty Compose capacity（历史） | evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`，最终 18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1229 audit，0 question error/drift/duplicate/PEL/lag，expected Worker=2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | +| 实现提交 / PR | clean commit `9a20676dcf545040782f04c166205d0043345753` 已普通 push 至 `codex/complete-evaluation-workflow`；[PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 head 精确指向该 SHA | +| Clean-SHA Compose acceptance | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`；`repository.commit=9a20676dcf545040782f04c166205d0043345753`、`dirty=false`、`status=passed`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和 cleanup C/V/N empty 均通过 | +| Clean-SHA Compose capacity | evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；`repository.commit=9a20676dcf545040782f04c166205d0043345753`、`dirty=false`、`status=passed`；1W/2W/burst wall `8.255963/4.628834/6.428385s`、`7.267474/12.962228/9.333604 q/s`，18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1230 audit，0 question error/drift/duplicate/PEL/lag；lease-owner SIGKILL 恢复场景有 1 个预期 failed attempt，expected Worker=2，cleanup C/V/N/image=0 且 image `1/1/0/0`；offline Mock、非 SLO | +| 实现提交前 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file index 为 0 Blocker/High/Medium；hydration/import integrity 修复目标集 `67 passed` | +| 实现精确 SHA 远程 CI | [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对 `9a20676dcf545040782f04c166205d0043345753` 的 backend、backend-integration、full-stack reliability 与 frontend 4/4 必需 job 全部成功 | +| 仍待门禁 | 形成证据文档 commit，普通 push 并等待该文档精确 SHA 的四个必需 CI job 全部成功 | +| 状态边界 | P2-06 repository closeout=`pending_on_docs_ci`；Phase 2=`in_progress`；P2-07=`not_started` | 下表是 2026-08-25 Phase 2 可靠性切片的历史基线，不是本次标准数据/真实评测 CLI 提交的通过声明。本次实际数量必须在当前工作日志和精确 commit 的 CI 中重新记录，不能沿用这些数字： diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 9b621bf..e9426e2 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -20,7 +20,7 @@ ## 当前功能范围 -- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容、双方言 Alembic 链在当前 P2-06 工作树扩展至 `20260828_0005`。 +- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容、双方言 Alembic 链已在 P2-06 implementation SHA `9a20676…` 扩展至 `20260828_0005`。 - Redis at-least-once 通知；Run、取消、重试、租约、Response、终态、治理、attempt ledger 和 audit 全由数据库裁决。 - 原子 claim、数据库时间 lease/heartbeat、fencing、有限 retry/backoff、取消、过期接管、duplicate no-op 和 dead-letter。 - 停写只读 SQLite→空 PostgreSQL 的单向 importer；`0005` 按依赖顺序复制 13 张应用表并做 count/PK/content fingerprint，源有 live Worker generation 时拒绝，stopped/stale progress 可精确复制。keyring 仍在数据库之外。 @@ -28,7 +28,7 @@ - 每个 Provider HTTP attempt 的 reserve→send-started→actual/conservative settlement 或 confirmed pre-send release never-delete ledger；materialized scope/bucket counter 仅作投影，任何高/低漂移 fail closed。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model due ordering。 - typed audit、分页 Run audit、task history counters、基于 Run 数据库时间戳的 queue/execution/end-to-end latency、严格规范化 Provider metadata 和非秘密 credential audit。 -- 固定低基数 Prometheus text exporter、八条仓库内告警规则/Runbook、DB-time Worker generation/progress 聚合、canonical audit archive/离线 verify/reconcile/restore/delete 和全日志源治理；这些 P2-06 功能已实现，仓库级全门禁与远程门禁仍待收尾。 +- 固定低基数 Prometheus text exporter、八条仓库内告警规则/Runbook、DB-time Worker generation/progress 聚合、canonical audit archive/离线 verify/reconcile/restore/delete 和全日志源治理；这些 P2-06 功能已进入 clean implementation SHA 并通过 clean Compose 与远程实现门禁，仓库级收尾只待证据文档提交自身的精确 SHA CI。 - Run Detail 展示 managed/delayed/exhausted、治理原因和明确 UTC not-before；旧 Run 与可信本地 CLI 明确为 `legacy_unmanaged`。 - 真实 PostgreSQL 竞争测试及 Mock-only enhanced capacity/acceptance；精确实现 SHA 的完整 capacity、9/9 acceptance、`P2-local-control-plane-v2` 多轮单机资格与远程 4/4 CI 已通过。 @@ -61,12 +61,12 @@ | ID | 状态 | 已交付与剩余范围 | | --- | --- | --- | | P2-01 一致性与容量设计 | `completed` | ADR-0012～0014、DB truth/lease/fencing/治理、v2 四 cell 多轮统计、恢复与连接模型已交付；clean SHA `b6a35fe…` 的 1+5 资格为 23/23、`qualified`；证据文档 commit `875f13a…` 已 push，精确 SHA CI 4/4 成功 | -| P2-02 PostgreSQL 迁移 | `slice_delivered` | 历史 `0002`～`0004` 与 12 表 importer 已通过精确 SHA 远程门禁；当前工作树增加 `0005` / 13 表 importer、live Worker preflight 和 populated downgrade guard，待 P2-06 最终门禁 | -| P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；当前工作树实现 generation 级 DB-time scan/claim/lease-heartbeat/progress 与 stale 聚合，dependency probe 仍只表示 capability | +| P2-02 PostgreSQL 迁移 | `slice_delivered` | 历史 `0002`～`0004` 与 12 表 importer 已通过精确 SHA 远程门禁；`9a20676…` 增加 `0005` / 13 表 importer、live Worker preflight 和 populated downgrade guard，clean Compose 与远程实现门禁已通过 | +| P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；`9a20676…` 增加 generation 级 DB-time scan/claim/lease-heartbeat/progress 与 stale 聚合，dependency probe 仍只表示 capability | | P2-04 生命周期可靠性 | `foundation_delivered` | retry/backoff、取消、恢复、dead-letter、Response 幂等和三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；Provider 外部副作用仍为 at-least-once | | P2-05 并发治理 | `slice_delivered` | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序、counter 重算 fail-closed 与 ADR-0011 已实现；精确 SHA 的真实 PG/capacity/acceptance/CI 候选门禁已通过 | -| P2-06 可观测性 | `in_progress` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已实现；lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审已通过，commit/clean-SHA Compose/push 与精确 SHA CI 仍未完成 | -| P2-07 验证与运维 | `slice_delivered` | enhanced capacity/PG tests、正式 v2 单机资格、Operations/Performance/Deployment 与精确 evidence 已交付；backup/restore、完整失败矩阵与告警响应仍未完成 | +| P2-06 可观测性 | `pending_on_docs_ci` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已进入 clean commit `9a20676…`；clean capacity/9/9 acceptance 与该实现 SHA 的 CI 4/4 已通过，只待证据文档提交自身的精确 SHA CI | +| P2-07 验证与运维 | `not_started` | 既有 enhanced capacity/PG tests、正式 v2 单机资格与运维文档是共享前置证据，不代表本切片已启动；PostgreSQL+keyring backup/restore、Redis 重建、完整失败矩阵与告警响应尚未实施 | `slice_delivered` 表示该垂直切片及其候选门禁已交付，不表示整个阶段完成。Phase 2 必须保持 `in_progress`。 @@ -80,7 +80,7 @@ - [x] **crash seam 验收**：`reserved`→send-start、`send_started`→settlement、Response commit→最终恢复三条 deterministic DB seam injection 在完整 Compose acceptance 通过；它们不冒充精确时刻 `SIGKILL`。 - [x] **远程实现门禁**：GitHub Actions run [`33099260233`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33099260233) 对精确实现 SHA 4/4 成功。 - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 -- [ ] **P2-06 仓库级闭环未通过**：范围内功能与 lint/test/smoke、真实 PostgreSQL/Redis、双方言 migration、frontend build、Compose config、八规则 promtool、dirty capacity/9/9 acceptance 和修复后 staged 技术/安全终审已通过；仍须形成独立 commit、在 clean SHA 重跑 Compose、push 并等待该精确 SHA CI 全绿。 +- [ ] **P2-06 仓库级闭环待证据文档 CI**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [GitHub Actions run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 已通过；只待本次证据文档提交、push 与该文档精确 SHA CI 全绿。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -88,8 +88,8 @@ | 验证 | 实际结果 | 限制 | | --- | --- | --- | | `make lint` | 最新本地冻结树通过 | 同一实现 SHA 远程 lint/test 通过 | -| 最新本地 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现冻结树通过 | -| 最新真实 PostgreSQL/Redis integration | `29/29 passed` | 本地通过；同一实现 SHA 远程 integration 通过 | +| P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见下方独立行 | +| P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见下方独立行 | | `make smoke` | `1 passed, 7 deselected`，仅 Mock | 最新本地冻结树通过；未调用真实 Provider | | 定向治理/API/Worker | 目标套件零失败；独立审计记录 `218 passed`；完整性边界集合 `18 passed` | 已由最终全量、真实 integration 与候选 evidence 补充 | | SQLite/PostgreSQL Alembic | 隔离 SQLite 与临时 PostgreSQL 16 的 prepare/upgrade/downgrade/upgrade/check 通过 | 本地通过；精确实现 SHA 的远程 integration 亦通过 | @@ -97,16 +97,19 @@ | enhanced capacity | `665244e…` 上通过；evidence SHA-256 `40deadeb…0588` | 有限 policy、4×202/2×429、yield/fairness/fault/reconciliation；Mock-only 非 SLA | | full Compose acceptance | `665244e…` 上 9/9；evidence SHA-256 `ab311665…ddec` | 三条 deterministic seam 与 cleanup 均通过 | | 正式 v2 单机资格 | `b6a35fe…` 上 1+5、23/23；aggregate SHA-256 `a76d167b…d0d9` | 每轮 22/330/330/331、hard invariant 与 exact-project cleanup 通过；Mock-only 非生产 SLA | -| 远程 CI | run `33146681285` 4/4 | 精确 v2 实现 SHA 全绿；PR #2 未合并 | +| P2-01 远程 CI | run `33146681285` 4/4 | 精确 v2 实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | 设计/计时修复远程 CI | SHA `1cd19c51ed309316047a18ed3b2a308647af495d`，run `33081854406`，4/4 | 不包含当前治理实现 | -| P2-06 lint/test | `make lint` 全绿（Ruff 152 files、ESLint、TS）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed` | 当前未提交工作树通过；只用 Mock/Stub | -| P2-06 smoke/integration/migration/build/config | smoke `1 passed, 7 deselected`；临时 PG16/Redis7 migration/check 后 integration `33 passed, 0 skipped`；临时 SQLite head→`0001`→head/check、frontend build（2192 modules，保留 662.39 kB warning）、Compose config 全绿 | 默认用户 SQLite 未到 head且未擅自迁移；首次 integration cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；当前结果尚未绑定远程 SHA | +| P2-06 lint/test | `make lint` 全绿（Ruff 152 files、ESLint、TS）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed` | clean implementation commit 前冻结树通过；只用 Mock/Stub | +| P2-06 smoke/integration/migration/build/config | smoke `1 passed, 7 deselected`；临时 PG16/Redis7 migration/check 后 integration `33 passed, 0 skipped`；临时 SQLite head→`0001`→head/check、frontend build（2192 modules，保留 662.39 kB warning）、Compose config 全绿 | 默认用户 SQLite 未到 head且未擅自迁移；首次 integration cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；实现 SHA 的远程同类门禁已通过 | | P2-06 规则门禁 | 临时 `prom/prometheus:v3.5.0` 容器中 `promtool check rules` 成功 | 八条规则全部通过；不表示仓库部署 Prometheus/Alertmanager | | P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`；`0005` populated 与 isolated `0004` refusal、两层空库往返、cleanup C/V/N empty 均通过 | | P2-06 dirty capacity | 最新 artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453` 通过 | 18 Runs/270 Responses/270 question executions/271 reservations/1229 audit；0 error/drift/duplicate/PEL/lag，expected Worker 2、cleanup C/V/N/image 0；offline Mock、非 SLO | +| P2-06 clean acceptance | `9a20676…` 上 9/9；artifact `llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，`dirty=false` | Worker `2/2/2/0/0`；两级 populated refusal、两层空库往返、cleanup C/V/N empty | +| P2-06 clean capacity | `9a20676…` 上通过；artifact `llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`，`dirty=false` | 1W/2W/burst `7.267474/12.962228/9.333604 q/s`；18/270/270/271/1230，0 question error/drift/duplicate/PEL/lag，expected Worker 2、cleanup C/V/N/image 0；offline Mock、非 SLO | +| P2-06 implementation remote gate | PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) | 精确 `9a20676dcf545040782f04c166205d0043345753`，四个必需 job 全 success | | P2-06 补充 Ruff | 过宽 scripts 命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | 保留首次结果，不扩大本切片清理范围 | -| P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 当前 dirty 工作树终审通过 | -| P2-06 剩余门禁 | commit、clean-SHA Compose、push/精确 SHA CI 待完成 | 不得借用上述历史绿色结果；保持 `in_progress` | +| P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | +| P2-06 剩余门禁 | 本次证据文档 commit、push 与该文档精确 SHA CI | 实现/clean-SHA Compose/implementation CI 已完成；完成文档门禁前状态为 `pending_on_docs_ci` | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 @@ -125,15 +128,15 @@ | 限额并发突破 | canonical scope、固定锁序、DB transaction、ledger 重算、真实 PG integration 与 v2 多轮资格 | 超出固定单机 profile 时重新测量并持续回归 | | Provider 调用/费用重复 | send-start marker、保守结算、本地幂等及三条 crash seam acceptance | 外部 exactly-once 不可承诺 | | 长 Run 饥饿 | finite quantum、due ordering、dispatch/failure 分离及 v2 每轮公平性硬门禁 | 更大规模或不同 Worker 拓扑需重新建模 | -| Worker 停滞不可见 | DB-time progress/liveness 聚合、exporter 与 `WorkerStalled` rule；dependency probe 保持 capability-only | 完成 P2-06 门禁，并在 P2-07 演练告警处置/扩缩 | -| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination、canonical archive/verify/精确 delete/restore | 完成 P2-06 门禁；P2-07 验证异地存储与整库恢复边界 | +| Worker 停滞不可见 | DB-time progress/liveness 聚合、exporter 与 `WorkerStalled` rule；dependency probe 保持 capability-only | 完成 P2-06 文档门禁，并在 P2-07 演练告警处置/扩缩 | +| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination、canonical archive/verify/精确 delete/restore | 完成 P2-06 文档门禁；P2-07 验证异地存储与整库恢复边界 | | 容量结论过度外推 | Mock-only、环境/config/evidence 指纹、1+5 多轮和明确支持 profile | 不冒充 Provider/生产 SLA；环境或 profile 变化必须重新资格 | | 灾难恢复失败 | 13 表 importer、迁移 guard、audit archive 自身 restore、独立 keyring 边界 | PostgreSQL backup/restore、keyring 配对、Redis 重建与完整恢复演练 | ## 交付物与下一任务 -已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。当前 P2-06 工作树另已实现 `0005` / 13 表 importer、Worker DB-time progress、exporter/八规则、audit retention CLI 与日志治理，dirty capacity/9/9 acceptance 和最终 review 已通过；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 先完成 commit/clean-SHA Compose/精确 SHA CI，再推进 P2-07 backup/restore 和剩余运维演练。 +已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 已包含 `0005` / 13 表 importer、Worker DB-time progress、exporter/八规则、audit retention CLI 与日志治理，并通过 clean capacity/9/9 acceptance 和远程 4/4 CI；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 完成本次证据文档提交自身的精确 SHA CI，再独立启动 P2-07 backup/restore 和剩余运维演练。 ## 状态 -`in_progress`。P2-01 已完成，P2-05 已交付；P2-06 功能已实现但仓库级门禁未闭环，P2-07 backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +`in_progress`。P2-01 已完成，P2-05 已交付；P2-06 实现与 clean-SHA 远程门禁已通过、仓库级状态为 `pending_on_docs_ci`，P2-07 尚未启动。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 diff --git a/docs/plans/2026-08-28-phase-2-observability-retention.md b/docs/plans/2026-08-28-phase-2-observability-retention.md index 51e0cf8..b0248a0 100644 --- a/docs/plans/2026-08-28-phase-2-observability-retention.md +++ b/docs/plans/2026-08-28-phase-2-observability-retention.md @@ -1,7 +1,7 @@ # Phase 2 可观测性与审计保留执行计划 - Owner: Codex -- Status: active +- Status: active (`pending_on_docs_ci`) - Created: 2026-08-28 - Updated: 2026-08-28 - Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) @@ -10,7 +10,7 @@ ## Context -Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audit/history 和单机 Mock 控制面资格。本计划开始时，数据库 API 能返回任务 gauges、typed event counters 和 Run latency，但没有稳定低基数 exporter/规则、Worker 主循环进展或可执行 audit retention。当前工作树已经实现这些 P2-06 能力，lint/test/smoke/integration/migration/build/config/rules、dirty capacity/9/9 acceptance 与修复后 staged 技术/安全终审也已通过；计划仍保持 active，因为独立 commit、clean-SHA Compose、push 和该精确 SHA 的远程 CI 尚未完成。 +Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audit/history 和单机 Mock 控制面资格。本计划开始时，数据库 API 能返回任务 gauges、typed event counters 和 Run latency，但没有稳定低基数 exporter/规则、Worker 主循环进展或可执行 audit retention。P2-06 实现已进入 clean commit `9a20676dcf545040782f04c166205d0043345753`，clean-SHA capacity/9/9 acceptance 与该实现 SHA 的远程 CI 4/4 均通过；计划仍保持 active，因为本次证据文档提交及其自身精确 SHA CI 尚未完成。 ## Objective @@ -71,7 +71,7 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi - Logging: 生产 logger 消息必须为无格式参数的字面量；第三方动态消息统一固定化，Uvicorn raw access log 禁用。 6. [in_progress] 执行完整验证、终审和远程门禁。 - Files/modules: 全部本切片文件与状态文档。 - - Validation: 定向+全量+lint+smoke+双方言 migration/PG integration+Compose/规则/secret/diff；dirty capacity/acceptance 与修复后 staged 技术/安全终审已通过；独立 commit 后 clean-SHA Compose、push、精确 SHA CI 4/4。 + - Validation: 定向+全量+lint+smoke+双方言 migration/PG integration+Compose/规则/secret/diff、clean-SHA capacity/acceptance 与实现 SHA CI 4/4 已通过；剩余门禁仅为证据文档提交、push 和该文档 SHA 自身 CI 4/4。 ## Risks @@ -98,12 +98,15 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi | Frontend build | `cd frontend && npm run build` | production build 成功 | 成功，2192 modules；662.39 kB 主 chunk warning 为非阻断既有告警。首次在仓库根误运行 `npm run build` 因无 `package.json` 失败，随后使用正确目录通过 | | Deployment | `docker compose config --quiet` | exit 0 | exit 0 | | Alert rules | unit test + `promtool check rules` | 八条规则可被 Prometheus 解析且只引用已交付指标 | 临时 `prom/prometheus:v3.5.0` 验证八条规则全部成功 | -| Dirty Compose acceptance | `make phase2-acceptance` | 9/9、Worker `2/2/2/0/0`、两级 populated refusal/两层空库往返、cleanup empty | 通过；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | +| Dirty Compose acceptance | `make phase2-acceptance` | 9/9、Worker `2/2/2/0/0`、两级 populated refusal/两层空库往返、cleanup C/V/N empty | 通过；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | | Dirty Compose capacity | `make phase2-capacity` | 最新脚本全通过且 cleanup empty | artifact `llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`；18/270/270/271/1229，0 error/drift/duplicate/PEL/lag，Worker expected 2，cleanup C/V/N/image=0；offline Mock、非 SLO | +| Clean-SHA Compose acceptance | `make phase2-acceptance` | exact clean implementation SHA、9/9 与 cleanup C/V/N empty | `9a20676…` 上通过；artifact `llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，`dirty=false`；Worker `2/2/2/0/0`、两级 populated refusal、两层空库往返、cleanup C/V/N empty | +| Clean-SHA Compose capacity | `make phase2-capacity` | exact clean implementation SHA、完整对账与 cleanup empty | `9a20676…` 上通过；artifact `llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst `7.267474/12.962228/9.333604 q/s`，18/270/270/271/1230，0 question error/drift/duplicate/PEL/lag，cleanup C/V/N/image=0；offline Mock、非 SLO | | Scripts Ruff | 目标 correctness/import 规则 | P2-06 脚本无 E/F/I 问题 | 过宽默认命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | -| Staged 技术/安全终审 | Blocker/High/Medium 终审 + 受影响回归 | 无未解决 B/H/M | structured-extra High 与 Worker `__main__` logger Medium 已修复；最新 76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | +| Implementation staged 技术/安全终审 | Blocker/High/Medium 终审 + 受影响回归 | 无未解决 B/H/M | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | | Docs/security review | Markdown 链接、secret/diff scan | 无坏链接、秘密或虚假状态 | 状态文档相对链接、whitespace diff、staged secret/path/blob 扫描均通过 | -| Remote gate | push 后 GitHub Actions exact SHA | 四个必需 job 全 success | 待执行 | +| Implementation remote gate | push 后 GitHub Actions exact SHA | 四个必需 job 全 success | PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)；[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对 `9a20676dcf545040782f04c166205d0043345753` 4/4 success | +| Evidence-doc remote gate | 本次文档 commit 的 GitHub Actions exact SHA | 四个必需 job 全 success | 待本次文档提交、push 后执行；完成前 P2-06 保持 `pending_on_docs_ci` | ## Rollback @@ -120,9 +123,10 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi ## Completion evidence - Changed files: `0005`/models/importer/prepare、Worker service/runner/progress/probe、JSON/Prometheus API 与 collector/renderer、audit validator/archive/retention CLI、logging/config/Compose、规则与相关测试/文档。 -- Commands run: `make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 与 integration `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、正确目录 frontend build、`docker compose config --quiet`、Prometheus `v3.5.0` 八规则校验、dirty capacity 和 9/9 acceptance 成功。 -- Acceptance evidence: dirty artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker `2/2/2/0/0`、两级 populated refusal、两层空库往返与 cleanup empty。最新 dirty capacity artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，0 error/drift/duplicate/PEL/lag 且 cleanup empty；offline Mock、非 SLO。 -- Not run: 当前尚未完成最终文档/秘密/diff 检查、clean-SHA Compose 和精确 SHA CI。默认用户 SQLite 未迁移到 head，按保护原则保留原状。 +- Commands run: `make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 与 integration `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、正确目录 frontend build、`docker compose config --quiet`、Prometheus `v3.5.0` 八规则校验、clean-SHA capacity/9/9 acceptance 成功。 +- Acceptance evidence: implementation commit `9a20676dcf545040782f04c166205d0043345753` 的 clean acceptance artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`；clean capacity artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`。两者 `dirty=false`、status passed；acceptance cleanup C/V/N empty，capacity cleanup C/V/N/image empty。Capacity 是 offline Mock 非 SLO。 +- Remote evidence: implementation commit 已 push 到 `origin/codex/complete-evaluation-workflow`；PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对该精确 SHA 4/4 success。 +- Not run: 本次证据文档提交尚未形成，因此尚无该文档精确 SHA 的 CI；默认用户 SQLite 未迁移到 head，按保护原则保留原状。 - Known issues: P2-07 仍为下一独立切片；Phase 2 保持 `in_progress`。 ## Decision and discovery log @@ -138,4 +142,5 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi | 2026-08-28 | implementation | `app.db` 与 `app.governance` runtime exports 改为 lazy，CLI `verify` fresh process 不再读取 DB 配置或创建 engine/目录。 | 保持 archive verification 真正离线，同时保留既有 public imports。 | | 2026-08-28 | review | importer committed-target integrity、retention no-op postverify、外部 logger extra、FIFO/line cap 与 PostgreSQL advisory/row-lock 门禁经终审补强并进入回归。 | 相关目标测试、全量与真实 integration 已通过。 | | 2026-08-28 | gate | Prometheus v3.5.0 八规则、dirty capacity 与 9/9 acceptance 通过；structured-extra High 与 Worker `__main__` logger Medium 修复后，最新 staged 技术/安全终审为 0 Blocker/High/Medium。 | acceptance `11554c25ec2d` / `d5f058…9fde`；capacity `c6de062ab77e` / `4aeb827…8453`；clean-SHA 仍须重跑。 | -| 2026-08-28 | status | P2-06 功能与 dirty 本地门禁已完成，但 commit/clean-SHA Compose/push 与精确 SHA CI 尚未完成。 | P2-06/Phase 2 均保持 `in_progress`；先完成步骤 6，再启动 P2-07。 | +| 2026-08-28 | gate | clean implementation commit `9a20676…` 的 capacity/9/9 acceptance 与远程 run `33164609388` 4/4 通过。 | clean artifacts 为 acceptance `92e173eeee28` / `e4ffb866…0fc47`、capacity `ca5673061b0f` / `2382f913…f5039`；不复用 dirty evidence冒充 clean gate。 | +| 2026-08-28 | status | P2-06 实现与 clean-SHA/implementation-CI 门禁已完成；证据文档提交及其自身 exact-SHA CI 尚未完成。 | P2-06=`pending_on_docs_ci`、Phase 2=`in_progress`、P2-07=`not_started`；先完成步骤 6 再启动 P2-07。 | diff --git a/docs/worklogs/2026-08-28-phase-2-observability-retention.md b/docs/worklogs/2026-08-28-phase-2-observability-retention.md index a737fb1..494a831 100644 --- a/docs/worklogs/2026-08-28-phase-2-observability-retention.md +++ b/docs/worklogs/2026-08-28-phase-2-observability-retention.md @@ -10,7 +10,7 @@ - 关联计划：[Phase 2 可观测性与审计保留执行计划](../plans/2026-08-28-phase-2-observability-retention.md) - 决策基础：[ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](../decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) - 本轮决策：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) -- 当前状态：`in_progress`；P2-06 单独推进，Phase 2 仍为 `in_progress` +- 当前状态：`pending_on_docs_ci`；P2-06 implementation/clean-SHA/remote gate 已通过，证据文档精确 SHA CI 尚待；Phase 2 仍为 `in_progress` ## 目标与背景 @@ -117,7 +117,7 @@ P2-01 已在精确提交与远程 CI 上闭环，现有系统也已经提供 Pos ### 状态边界 -- 以上功能已进入当前工作树，lint/test/smoke/integration/migration/build/config/rules 与 dirty capacity/9/9 acceptance 已通过；后续 staged security review 发现 structured logging High，继续复核又发现 `python -m app.worker` logger Medium，两项修复与定向回归均已落盘。最新 76-file staged 技术/安全终审重新收敛为 0 Blocker/High/Medium。独立 commit、clean-SHA Compose、push 与精确 SHA GitHub Actions 尚未完成，因此 P2-06 仓库级状态保持 `in_progress`。 +- 以上功能已进入 clean implementation commit `9a20676dcf545040782f04c166205d0043345753`。lint/test/smoke/integration/migration/build/config/rules、clean-SHA capacity/9/9 acceptance、push 与该实现 SHA 的 GitHub Actions 4/4 均已通过。staged security review 发现的 structured logging High 与 `python -m app.worker` logger Medium 已修复，最终技术/安全终审为 0 Blocker/High/Medium。当前只待本次证据文档提交及其自身精确 SHA CI，因此 P2-06 仓库级状态为 `pending_on_docs_ci`。 - P2-06 archive restore 只恢复 typed audit archive 自身。PostgreSQL+keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置与剩余故障矩阵仍属于 P2-07；Phase 2 继续 `in_progress`。 ## 实际命令与结果 @@ -145,14 +145,17 @@ P2-01 已在精确提交与远程 CI 上闭环，现有系统也已经提供 Pos | 过宽 Ruff scripts 命令 | 报告 93 条既有 modernization 告警；随后按本任务静态合同运行 `--select E,F,I` 并通过，未修改范围外历史现代化问题 | | `make phase2-acceptance`（dirty 工作树） | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`；application populated `0005` refusal、isolated populated `0004` refusal、两层空库往返、cleanup containers/volumes/networks empty | | `make phase2-capacity`（最新 dirty 工作树） | artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`；18 Runs/270 Responses/270 QuestionExecutions/271 reservations（270 actual + 1 conservative）/1229 audit，0 error/drift/duplicate/PEL/lag，Worker expected 2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | +| `make phase2-acceptance`（clean SHA） | implementation commit `9a20676dcf545040782f04c166205d0043345753`，artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，`dirty=false`、9/9；Worker `2/2/2/0/0`，两级 populated refusal、两层空库往返、queue 0/0 与 cleanup C/V/N empty；脚本不承诺 build image cleanup | +| `make phase2-capacity`（clean SHA） | 同一 implementation commit，artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`，`dirty=false`；1W/2W/burst wall `8.255963/4.628834/6.428385s`、`7.267474/12.962228/9.333604 q/s`；18/270/270/271/1230，0 question error/drift/duplicate/PEL/lag，expected Worker=2、stalled/shortfall=0，故障恢复后的瞬时 registered/live=3；cleanup C/V/N/image=0 且 image `1/1/0/0`；offline Mock、非 SLO | +| implementation commit / push / exact-SHA CI | commit `9a20676dcf545040782f04c166205d0043345753` 已普通 push 至 `origin/codex/complete-evaluation-workflow`；PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)；[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 精确绑定该 SHA，Frontend、Compose reliability、Backend、PG/Redis integration 四个 job 全 success | | 先前 code review 与 hydration/import integrity 回归 | 当时结论为 0 Blocker/High/Medium、目标集 `67 passed`；其后 staged security review 发现 structured logging High 与 Worker entrypoint Medium，均已修复并重新复核 | -| 最新 staged 技术/安全终审 | 76-file index 为 0 Blocker/High/Medium；独立 logging/source/queue `19 passed`，archive/retention/CLI/exporter/Docker-script `167 passed`；secret/path/blob/category 扫描无真实凭据或禁止产物 | +| Implementation staged 技术/安全终审 | 76-file implementation index 为 0 Blocker/High/Medium；独立 logging/source/queue `19 passed`，archive/retention/CLI/exporter/Docker-script `167 passed`；secret/path/blob/category 扫描无真实凭据或禁止产物 | | README、TESTING、CHANGELOG、PROJECT_STATUS、ROADMAP、Phase 2、NEXT_TASK、计划/工作日志状态同步 | 已把实现事实与 pending-gate/P2-07 边界写入当前 diff；本轮链接/diff 检查通过 | | 九份状态文档相对链接检查 | 全部目标存在 | | 七份 tracked 状态文档 `git diff --check` + 新 plan/worklog 的 `git diff --no-index --check` | 无 whitespace error；no-index 命令因文件整体为新增返回差异状态 1，且没有 `--check` 诊断 | ## 已知问题与下一步 -- P2-06 三条实现线、structured logging High 与 Worker entrypoint Medium 修复已落地；lint、全量 test、smoke、frontend build、临时 SQLite/真实 PG migration、真实 PG/Redis integration、Compose config、八规则 promtool、dirty capacity/9/9 acceptance 与最新 staged 技术/安全终审均已通过。 -- 全门禁通过后形成独立 commit，在 clean SHA 重跑 capacity/acceptance，普通 push 当前分支并等待该精确 SHA 四个必需 CI job 全绿；在此之前不得写成 P2-06 completed。 +- P2-06 三条实现线、structured logging High 与 Worker entrypoint Medium 修复已落地；lint、全量 test、smoke、frontend build、临时 SQLite/真实 PG migration、真实 PG/Redis integration、Compose config、八规则 promtool、clean-SHA capacity/9/9 acceptance、implementation push 与精确 SHA 4/4 CI 均已通过。 +- 当前只需完成本次 evidence-doc commit、普通 push 与该文档精确 SHA 四个 CI job；绿色前 P2-06 为 `pending_on_docs_ci`，不得写成 completed。绿色后另做状态收尾提交并等待其自身 CI。 - P2-07 未开始，仍缺数据库+keyring 配对 backup/restore、Redis 重建、告警响应与完整恢复矩阵；Phase 2 继续 `in_progress`。 From fc7bc9bf406c15920aede0be08bb128c25da277f Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Fri, 28 Aug 2026 19:15:26 +0800 Subject: [PATCH 03/12] docs: close phase 2 observability task --- CHANGELOG.md | 6 ++-- README.md | 8 ++--- docs/NEXT_TASK.md | 33 +++++++++---------- docs/PROJECT_STATUS.md | 18 +++++----- docs/ROADMAP.md | 18 +++++----- docs/TESTING.md | 14 ++++---- docs/phases/PHASE-2-RELIABILITY.md | 18 +++++----- ...6-08-28-phase-2-observability-retention.md | 18 +++++----- ...6-08-28-phase-2-observability-retention.md | 11 ++++--- 9 files changed, 73 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c244b4c..a04abd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Added -- Added the P2-06 implementation, now `pending_on_docs_ci`, under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. +- Added the completed P2-06 implementation under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. - Added Alembic revision `20260828_0005`, the `worker_processes` generation table, bounded `(expires_at,id)` / `(occurred_at,id)` audit scan indexes, and DB-time coalesced Worker scan/claim/lease-heartbeat/progress recording. `/tasks/metrics` and the exporter expose only low-cardinality aggregates; the dependency probe remains capability-only. The stopped SQLite-to-PostgreSQL importer now copies and reconciles all 13 application tables, rejects a still-live Worker generation, and preserves stopped/stale process facts. - Added `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete`. The v1 canonical JSONL archive is bounded, exact-digest bound, atomically installed without replacement at `0600`, and independently verifiable without a database engine; delete is never implicit, restore/delete require an exact confirmed archive SHA-256, and commit-unknown outcomes require read-only reconciliation before retry. These integrity hashes are neither signatures nor WORM guarantees. - Added the Phase 2 single-host control-plane qualification under ADR-0012 through ADR-0014. The `P2-local-control-plane-v2` harness fixes one warm-up plus exactly five measured trials, four Mock-only capacity cells, paired Worker scaling, queue/execution/end-to-end and recovery objectives, ledger-derived correctness gates, bounded capacity formulas, and scoped project-image cleanup. Clean implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` passed 23/23 SLO checks with all six child invariants and cleanup checks; the earlier v1 aggregate remains permanently `unqualified` rather than being discarded or reclassified. This qualifies only the recorded single-host Mock control-plane profile, not production SLA, real-Provider capacity, HA, billing, or external exactly-once behavior. @@ -107,11 +107,11 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Verification -- P2-06 implementation gates are complete. Commit [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) was pushed to `codex/complete-evaluation-workflow` and is tracked by [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3); exact-SHA GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) passed all four required jobs. Its local gates include green combined targeted suites, `make lint` across 152 Python files plus ESLint/TypeScript, backend `916 passed, 33 skipped`, frontend `38 passed`, offline Mock smoke `1 passed, 7 deselected`, PostgreSQL 16/Redis 7 migration/check and `33 passed, 0 skipped` integration, isolated SQLite migration/check, frontend build, Compose config, eight-rule Prometheus `v3.5.0` validation, and a 76-file technical/security review with 0 Blocker/High/Medium. Clean-SHA Compose acceptance passed 9/9 with `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` (SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`), Worker gauges `2/2/2/0/0`, and empty container/volume/network cleanup. Clean capacity passed with `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` (SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`): 1W/2W/burst QPS `7.267474/12.962228/9.333604`, wall `8.255963/4.628834/6.428385s`, 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit events, zero question error/drift/duplicate/PEL/lag, Worker expected 2 with shortfall 0, and empty container/volume/network/image cleanup with image counters `1/1/0/0`. The earlier dirty acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`) and dirty capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`, 1229 audit events) remain historical evidence rather than being overwritten. These are offline Mock observations, not production or real-Provider SLOs. P2-06 is `pending_on_docs_ci`, not complete: only this evidence-documentation commit and its own exact-SHA CI remain. Phase 2 stays `in_progress`, and P2-07 is `not_started`. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. +- P2-06 implementation gates are complete. Commit [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) was pushed to `codex/complete-evaluation-workflow` and is tracked by [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3); exact-SHA GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) passed all four required jobs. Its local gates include green combined targeted suites, `make lint` across 152 Python files plus ESLint/TypeScript, backend `916 passed, 33 skipped`, frontend `38 passed`, offline Mock smoke `1 passed, 7 deselected`, PostgreSQL 16/Redis 7 migration/check and `33 passed, 0 skipped` integration, isolated SQLite migration/check, frontend build, Compose config, eight-rule Prometheus `v3.5.0` validation, and a 76-file technical/security review with 0 Blocker/High/Medium. Clean-SHA Compose acceptance passed 9/9 with `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` (SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`), Worker gauges `2/2/2/0/0`, and empty container/volume/network cleanup. Clean capacity passed with `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` (SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`): 1W/2W/burst QPS `7.267474/12.962228/9.333604`, wall `8.255963/4.628834/6.428385s`, 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit events, zero question error/drift/duplicate/PEL/lag, Worker expected 2 with shortfall 0, and empty container/volume/network/image cleanup with image counters `1/1/0/0`. The earlier dirty acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`) and dirty capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`, 1229 audit events) remain historical evidence rather than being overwritten. These are offline Mock observations, not production or real-Provider SLOs. Evidence-documentation commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) was pushed, and its exact-SHA GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) passed all four required jobs, completing P2-06. Phase 2 stays `in_progress`; P2-07 is `ready/next` but not implemented. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. - Recorded two corrected local gate invocations without hiding their first outcomes: the initial integration cleanup command was rejected by the local safety policy before any container started, then the explicit cleanup/run passed; an over-broad Ruff invocation over `scripts/` surfaced 93 existing modernization warnings, while the intended `--select E,F,I` correctness/import gate passed. - On the governance/audit candidate at `665244e095905083b606b8e98e946ed1a02dc0fc`, `make test` passed with `604 passed, 29 skipped` on the backend and `38 passed` on the frontend; a separate real PostgreSQL/Redis integration run passed `29/29`. That frozen implementation also passed `make lint`, offline `make smoke` (`1 passed, 7 deselected`), the frontend production build, isolated SQLite and PostgreSQL migration/check gates, `docker compose config --quiet`, targeted governance/API/Worker suites, enhanced capacity, and 9/9 Compose acceptance. - Pushed the implementation and lock-order fixes through exact SHA `665244e095905083b606b8e98e946ed1a02dc0fc`. Its enhanced capacity evidence (`40deadeb…0588`) records finite policy, exact 4×`202` + 2×`429`, cooperative/cross-Model fairness, fault recovery, 18 Runs/270 Responses/271 reservations/1229 audit events, and empty cleanup; its 9/9 acceptance evidence (`ab311665…ddec`) includes all three deterministic database seams and empty cleanup. GitHub Actions run `33099260233` passed all four required jobs. At that gate, the formal SLO/exporter/retention/backup/Worker-progress closure was still pending. -- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress`; P2-06 is `pending_on_docs_ci`, and P2-07 is `not_started`. +- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress`; P2-06 is `completed`, and P2-07 is `ready/next` but not implemented. - Opened PR #1 for the trusted-local evaluation/Web credential/Run UX/SSE branch and obtained all four required GitHub Actions jobs for exact SHA `ab15862eab4870dda01fb079b44b509a7d737627` in run `33078921254`; this closes the previously documented no-PR/no-CI boundary for that SHA only. - Passed 205 backend non-infrastructure tests, five real PostgreSQL/Redis integration tests, 13 frontend tests, the isolated offline Mock smoke test, Ruff/ESLint/TypeScript checks, Vite production build, and SQLite/PostgreSQL Alembic upgrade/check/downgrade/upgrade gates. - Passed an isolated default-build Compose acceptance harness in all eight scenarios: topology/readiness, protocol-v1 baseline, API restart during execution, exact lease-owner SIGKILL and natural takeover, Redis stop/start with database reconciliation, pending cancellation, running cancellation plus duplicate delivery, and PostgreSQL `head -> 0001 -> head` protocol round-trip. diff --git a/README.md b/README.md index c14d227..eb352fc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ GitHub：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测工作台。它把模型注册、版本化 Benchmark、后台评测、逐题证据、汇总指标和排行榜放进一条可审计的本地流程，并以“默认离线、严格评分、结果可复现”为首要约束。 -当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选；P2-06 可观测性/审计保留切片处于 `pending_on_docs_ci`：实现及其精确 SHA 门禁已完成，仅剩本次 evidence closeout 文档 commit 自身的精确 SHA CI。PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger、typed audit 与 Worker progress 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 +当前版本在保留 SQLite 单机兼容路径的同时，已经交付 Phase 2 的可靠执行与治理候选；P2-06 可观测性/审计保留切片已完成实现、clean evidence 与 evidence-doc 精确 SHA CI，状态为 `completed`。PostgreSQL 是 Compose/部署目标和任务、四层治理、逐 HTTP attempt ledger、typed audit 与 Worker progress 的事实来源，Redis Streams 只提供可重复、可丢失的低延迟通知，独立 Worker 通过数据库租约和公平 question quantum 执行任务。完全不需要 API Key 的 Mock Demo 仍是默认验收路径；OpenAI-compatible Chat Completions 适配器及真实 Provider 调用始终是用户主动启用的可选能力。 ## 当前状态 @@ -18,8 +18,8 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - 可信本地 CLI 已提供 MMLU-Pro 与 GPQA-Diamond 的固定来源转换、真实 OpenAI-compatible 预检、可恢复执行和完整报告导出；这是 Phase 3 的客观题垂直切片，不代表 Phase 2 或 Phase 3 已完成。 - 自动化、CI、Compose 故障验收和容量演练的模型执行都只使用 Mock；根据层级使用临时 SQLite 或隔离 PostgreSQL 16/Redis 7，不访问真实模型服务，也不产生模型费用。 - Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 -- P2-06 当前状态为 `pending_on_docs_ci`，不能标记为 `completed`。实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已 push 到当前分支并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。绑定同一 clean SHA 的 Compose acceptance 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`），Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0` 且 cleanup C/V/N 全空；clean capacity evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`），1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。这是 Mock-only 单机观测，不是生产或真实 Provider SLO；此前 dirty evidence 继续保留为历史，不替代该 clean-SHA 结果。实现门禁已全部完成，仅剩本 evidence closeout 文档 commit 的 push 与精确 SHA CI。 -- P2-07 明确为 `not_started`；数据库/keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵仍未开始，因此 Phase 2 不能完成。Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 +- P2-06 状态为 `completed`。实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已 push 到当前分支并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。绑定同一 clean SHA 的 Compose acceptance 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`），Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0` 且 cleanup C/V/N 全空；clean capacity evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`），1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。这是 Mock-only 单机观测，不是生产或真实 Provider SLO；此前 dirty evidence 继续保留为历史，不替代该 clean-SHA 结果。Evidence-doc commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 [run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 成功，完成 P2-06 仓库级收尾。 +- P2-07 状态为 `ready/next`，但尚未实现；下一步才开始数据库/keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵，因此 Phase 2 仍为 `in_progress`。Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 最新、可复核的完成状态与测试证据以 [`docs/PROJECT_STATUS.md`](docs/PROJECT_STATUS.md) 和 [`docs/worklogs/`](docs/worklogs/) 为准；Roadmap 中的计划能力不等于已交付能力。 @@ -396,7 +396,7 @@ uv run llmbenchlab-audit-retention delete \ | --- | --- | --- | | Phase 0 | 项目治理、需求、架构、协议 | 已完成 | | Phase 1 | FastAPI + React + SQLite 的 MVP 垂直链路 | 已完成 | -| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 已闭环；P2-06 为 `pending_on_docs_ci`，实现 SHA `9a20676…` 已 push、PR #3 与实现 run `33164609388` 4/4、clean capacity/9/9 acceptance 全绿，仅剩 evidence-doc commit 自身精确 CI；P2-07 为 `not_started` | +| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 已闭环；P2-06 实现与 evidence-doc 精确 SHA CI 均全绿，状态为 `completed`；P2-07 为 `ready/next`，尚未实现 | | Phase 3 | 合规标准 Benchmark 与隔离代码评测 | MMLU-Pro/GPQA 客观数据切片已交付；IFEval、沙箱及其余验收未完成 | | Phase 4 | LLM Judge、人工校准与 Arena | 计划中 | | Phase 5 | Agent、工具调用与 Live Benchmark | 计划中 | diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 5e5978c..965ab98 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -1,14 +1,14 @@ -# 下一任务：完成 P2-06 证据文档 CI，再启动 P2-07 +# 下一任务：启动 P2-07 恢复与运维闭环 -> 状态：`pending_on_docs_ci`；P2-06 实现、clean-SHA capacity/9/9 acceptance、push 与 implementation SHA CI 已完成，只待本次证据文档提交自身的精确 SHA CI +> 状态：`ready`；P2-06 已完成仓库级闭环，下一步先建立 P2-07 独立计划、工作日志与必要 ADR > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) -> 当前计划：[Phase 2 可观测性与审计保留](plans/2026-08-28-phase-2-observability-retention.md) -> 当前日志：[2026-08-28 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) +> 已完成计划：[Phase 2 可观测性与审计保留](plans/2026-08-28-phase-2-observability-retention.md) +> 已完成日志：[2026-08-28 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) > 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) ## 当前事实 -P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 GitHub Actions [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，clean SHA 上的 `P2-local-control-plane-v2` 完成 1 warm-up + 5 measured、23/23 SLO，aggregate SHA-256 为 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`；证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。这些结果只属于对应历史 SHA，不能替代当前 P2-06 证据文档门禁。 +P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成：implementation SHA `9a20676dcf545040782f04c166205d0043345753` 的 clean-SHA capacity/9/9 acceptance 与 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确 4/4 通过。 当前 P2-06 已按 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 实现： @@ -19,19 +19,18 @@ P2-01 已完成，不再重复资格或“重跑碰绿”：实现 SHA `b6a35fef - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 提供 canonical JSONL v1、严格文件/权限/大小/schema/hash/rollup 校验、真正离线 verify、精确 digest 绑定、默认不删除和 commit-outcome 分类。它不提供签名/WORM，也不替代整库+keyring backup。 - 全生产日志调用受字面量/无格式参数静态门禁；第三方动态消息固定化、结构化数值有限化，raw Uvicorn access handler 关闭。 -当前验证事实：合并定向套件全绿；`make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 integration 为 `33 passed, 0 skipped`，隔离 migration、frontend build、Compose config 与 Prometheus v3.5.0 八规则校验均通过。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）在 implementation SHA `9a20676dcf545040782f04c166205d0043345753` 上 `dirty=false`、9/9，Worker `2/2/2/0/0`、两级 populated refusal、两层空库往返与 cleanup C/V/N empty。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）同样 `dirty=false`：1W/2W/burst `7.267474/12.962228/9.333604 q/s`，18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag；expected Worker=2、stalled/shortfall=0，故障恢复后的瞬时 registered/live=3，cleanup C/V/N/image=0。该结果是 offline Mock 非 SLO。structured-extra High 与 Worker `__main__` logger Medium 均已修复，最终技术/安全复核为 0 Blocker/High/Medium。实现提交已 push，PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 精确绑定该 SHA并4/4 success。默认用户 SQLite 尚未在 head且未擅自迁移；证据文档 SHA CI 绿色前 P2-06 仍不能写成 `completed`。 +P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。默认用户 SQLite 尚未在 head且未擅自迁移。 -## 立即执行：P2-06 证据文档收尾 +## 立即执行：建立 P2-07 工作包 -1. 复核本次证据文档 diff，确保 clean 与历史 dirty 结果不混用；capacity 的 `1 failed attempt` / `1 queue-notification error` 是注入故障的预期对账，不得笼统写成零，acceptance 也不得误写成清理了 build image。 -2. 检查 Markdown 链接、`git diff --check`、staged diff、秘密/凭据/raw evidence/本机路径/生成物；只提交文档，不修改实现或 evidence artifact。 -3. 形成独立 evidence-doc commit，普通 push 当前工作分支，通过 PR #3 触发 CI。 -4. 等待该**文档提交精确 SHA** 的四个必需 GitHub Actions job 全部 success。失败时读取日志、修复并对新 SHA 重跑；不得用实现 SHA 的绿色替代。 -5. 文档 SHA 绿色后，另做状态收尾提交：把 P2-06 标为 `completed`、计划步骤 6 标完成、NEXT_TASK 转为 P2-07；Phase 2 继续 `in_progress`。状态提交也须 push 并等待自身精确 SHA CI。 +1. 按 AGENTS/PLANS 先新建 P2-07 独立执行计划与工作日志，记录目标、非目标、验收、风险和实施步骤；不得复用 P2-06 日志冒充新任务记录。 +2. 完整勘察现有 PostgreSQL backup/restore、keyring、Redis consumer group、告警 Runbook 与 Compose/CI 边界；若改变恢复协议或安全取舍，实施前新增 ADR。 +3. 把 P2-07 拆成可独立验证的 backup/keyring、Redis rebuild、alert response 与故障矩阵步骤，先冻结恢复不变量和证据公开边界，再实施代码/脚本。 +4. 自动化只使用 Mock/Stub；任何数据库/volume/keyring 删除或替换必须使用隔离、精确目标和 fail-closed guard，不触碰默认用户数据。 ## P2-07：下一独立切片 -P2-07 当前为 `not_started`。P2-06 文档门禁全绿后，新建独立计划、工作日志和必要 ADR，完成： +P2-07 当前为 `ready`、尚未实现。新建独立计划、工作日志和必要 ADR 后，完成： - PostgreSQL backup → 空目标 restore → Alembic `20260828_0005` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 - 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 @@ -48,12 +47,12 @@ P2-07 当前为 `not_started`。P2-06 文档门禁全绿后，新建独立计划 ## Definition of Done -- P2-06：功能、文档、迁移与测试合同一致；全量本地/真实基础设施门禁通过；实现与证据文档各有独立 commit 并 push；对应精确 SHA 的四个必需 CI job 均全绿；状态/计划/日志记录完整且无秘密或生成归档。 -- Phase 2：只有 P2-06 上述门禁和 P2-07 backup/restore、Redis 重建、告警处置与完整故障矩阵均有可复核证据后，才可评估 `completed`。 -- 任一关键项未运行、失败或未形成精确 SHA 远程绿色时，相关任务不得标记 `completed`；当前 P2-06 保持 `pending_on_docs_ci`，Phase 2 保持 `in_progress`，且不得宣称生产 HA、灾难恢复 SLA、无限横向扩展、WORM 或 Provider exactly-once。 +- P2-06 已完成，不再重复其资格或证据门禁。 +- P2-07：backup/restore、Redis 重建、告警处置与约定故障矩阵必须有隔离真实 PostgreSQL/Redis、Mock-only Compose、秘密审查、独立 commit/push 和精确 SHA CI 证据，才能标记 completed。 +- Phase 2：只有 P2-07 也完成后才可评估 `completed`；在此之前保持 `in_progress`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展、WORM 或 Provider exactly-once。 ## 可直接复制给 Codex 的任务指令 ```text -继续执行 docs/NEXT_TASK.md。P2-06 implementation SHA 9a20676dcf545040782f04c166205d0043345753 的 clean capacity/9/9 acceptance 与 GitHub Actions run 33164609388 4/4 已通过；不要重跑碰绿，也不要提前启动 P2-07。现在复核并提交 clean evidence 文档，普通 push 当前工作分支并等待该文档精确 SHA 的四个 CI job 全绿。绿色后再做状态收尾提交，把 P2-06 标 completed、NEXT_TASK 转入新建 P2-07 计划；Phase 2 仍保持 in_progress。自动化只用 Mock/Stub，禁止公开 raw evidence 或真实凭据。 +继续执行 docs/NEXT_TASK.md。P2-06 已完成，不要重跑碰绿。现在按 AGENTS/PLANS 新建 P2-07 独立计划、工作日志和必要 ADR，先勘察并冻结 PostgreSQL+keyring 配对 backup/restore、Redis rebuild、八规则告警处置和剩余故障矩阵合同，再分步实现与验证。自动化只用 Mock/Stub，所有 destructive 操作只针对隔离、精确目标；Phase 2 在 P2-07 完成前保持 in_progress。 ``` diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index ec33cd6..7f09493 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -6,7 +6,7 @@ - Phase 0 — 项目治理和架构：`completed`（2026-08-24） - Phase 1 — MVP 垂直链路：`completed`（2026-08-25） -- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计与 P2-01 单机资格已完整交付；P2-06 为 `pending_on_docs_ci`，P2-07 为 `not_started`） +- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计、P2-01 单机资格与 P2-06 已完整交付；P2-07 为 `ready/next`，尚未实现） - Phase 3 — 标准 Benchmark 与代码评测：`in_progress`（仅可信本地 MMLU-Pro/GPQA-Diamond 客观题提前切片） - Phase 4–6：`planned` @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。P2-06 当前为 `pending_on_docs_ci`，只剩本次 evidence closeout 文档 commit 的 push 与精确 SHA CI，不能提前标记 `completed`。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。Phase 2 仍为 `in_progress`，P2-07 为 `ready/next`、尚未实现。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -41,19 +41,19 @@ - 精确 SHA `665244e…` 的增强 capacity 使用有限 policy、PostgreSQL 16、Redis 7 与两个 Worker 完成：并发 backlog 精确为 4 个 `202` + 2 个 typed `429`，cooperative yield 与跨 Model 公平顺序均有 durable audit 证据，最终 18 Runs/270 Responses/271 ledger/1229 audit 对账且无 active/reserved/overdrawn 漂移。 - P2-01 的 `P2-local-control-plane-v2` 已在干净 SHA `b6a35fe…` 从零执行 1 次 warm-up + 恰好 5 次 measured trial，本次 invocation 的 `discarded_trials=0`；四个 cell、23/23 SLO 与每轮 hard invariant 全部通过，容量模型为 `qualified`。aggregate SHA-256 为 `a76d167b…d0d9`，六轮均精确完成 22 Runs/330 Responses/330 QuestionExecutions/331 reservations，并清理本项目容器、卷、网络和唯一 build image。历史 v1 aggregate `f993c11f…e3b2` 继续保持 `failed/not_qualified`。 -## P2-06 实现（`pending_on_docs_ci`） +## P2-06 实现（`completed`） - [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；实现 SHA `9a20676dcf545040782f04c166205d0043345753` 将 Alembic head 扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 - `GET /api/v1/metrics/prometheus` 已实现固定 Prometheus text `0.0.4` gauge：一个 DB-time 读快照、15 分钟 typed-audit 窗口、1 小时 Run latency、硬读取上限、固定 enum label、整次 fail-closed 与每 API 进程 single-flight。`deploy/observability/` 提供固定八条规则和安全抓取示例；仓库不部署 Prometheus、Alertmanager 或通知发送器。 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 已实现 canonical JSONL v1、严格权限/大小/行/schema/hash/rollup 校验、离线 verify、精确 digest 绑定、默认不删除、双方言事务与 commit outcome 分类。Archive 是敏感运维文件，hash 只用于完整性/绑定，不是签名或 WORM，也不替代 P2-07 的数据库+keyring 备份。 - importer 当前合同为 `0005` 的 13 表精确 count/PK/content digest；live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 - 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 -- 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 +- 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。Evidence closeout 文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push，精确 SHA run `33165775037` 4/4 成功，P2-06 仓库级收尾完成。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 -## 仍未完成 +## 状态与后续 -- P2-06：状态为 `pending_on_docs_ci`，实现、clean-SHA Compose evidence、push 和实现精确 SHA CI 全部完成；只剩本 evidence closeout 文档 commit 的 push 与精确 SHA CI，因此仍不能标记 `completed`。 -- P2-07：状态为 `not_started`。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 +- P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 +- P2-07：状态为 `ready/next`，尚未实现。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 @@ -88,7 +88,7 @@ | P2-06 补充静态检查 | 过宽 `scripts/` Ruff 命令暴露 93 条既有 modernization 告警；`--select E,F,I` 通过 | 如实保留首次结果，不把范围外历史告警归为本次回归 | | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入实现 SHA `9a20676…` | | P2-06 实现远程 CI | PR #3；`9a20676dcf545040782f04c166205d0043345753` 的 run `33164609388` 4/4 | 实现精确 SHA 门禁完成 | -| P2-06 剩余门禁 | 本 evidence closeout 文档 commit 的 push 与精确 SHA CI | 当前 `pending_on_docs_ci`；不得标记 `completed` | +| P2-06 evidence 文档远程 CI | `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 run `33165775037` 4/4 | 精确文档 SHA 门禁完成；P2-06 为 `completed` | | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | @@ -117,4 +117,4 @@ ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 当前仅剩 evidence closeout 文档 commit 的 push 与精确 SHA CI；其通过前状态保持 `pending_on_docs_ci`。之后才可单独启动仍为 `not_started` 的 P2-07 数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 已完成仓库级收尾；下一任务为 `ready/next`、尚未实现的 P2-07 数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 3aa2e05..d4bac63 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,7 +30,7 @@ | --- | --- | --- | --- | --- | | Phase 0 | 项目治理和架构 | 可执行的需求、架构、协议、ADR 与持续文档流程 | `completed` | [PHASE-0-GOVERNANCE.md](phases/PHASE-0-GOVERNANCE.md) | | Phase 1 | MVP 垂直链路 | Mock 模型到 Run、逐题结果与排行榜的离线闭环 | `completed` | [PHASE-1-MVP.md](phases/PHASE-1-MVP.md) | -| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计与 P2-01 已交付；P2-06 为 `pending_on_docs_ci`，P2-07 为 `not_started` | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | +| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计、P2-01 与 P2-06 已交付；P2-07 为 `ready/next`，尚未实现 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | | Phase 3 | 标准 Benchmark 与代码评测 | 已有 MMLU-Pro/GPQA 可信本地切片；IFEval、沙箱与完整插件体系待完成 | `in_progress` | [PHASE-3-BENCHMARKS.md](phases/PHASE-3-BENCHMARKS.md) | | Phase 4 | Judge、Arena 与长上下文 | 可校准 Judge、Pairwise Judge、个人 Arena 和长上下文评测 | `planned` | [PHASE-4-JUDGE-ARENA.md](phases/PHASE-4-JUDGE-ARENA.md) | | Phase 5 | Agent、私有与 Live Benchmark | 工具调用轨迹、隔离私有集和持续更新的 Live Benchmark | `planned` | [PHASE-5-AGENT-LIVE.md](phases/PHASE-5-AGENT-LIVE.md) | @@ -206,8 +206,8 @@ | P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付；P2-06 实现 SHA `9a20676…` 增加 generation 级 DB-time progress，dependency probe 仍只检查 capability | | P2-04 生命周期可靠性 | 可靠基础已交付 | retry/取消/恢复/dead-letter/终态重算及三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；外部调用仍不保证 exactly-once | | P2-05 并发治理 | 切片已实现 | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序和完整性 fail-closed 已实现；真实 PG/capacity/acceptance/精确 SHA CI 候选门禁已通过 | -| P2-06 可观测性 | `pending_on_docs_ci` | exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现；`9a20676…` 已 push、PR #3、实现 run `33164609388` 4/4，clean capacity/9/9 acceptance 全绿；仅剩 evidence-doc commit 自身的精确 SHA CI，不能标记 `completed` | -| P2-07 验证与运维 | `not_started` | P2-01/P2-06 已提供 capacity/PG/acceptance 基线，但 P2-07 本身的数据库+keyring backup/restore、Redis 重建、告警响应与完整失败矩阵尚未开始 | +| P2-06 可观测性 | `completed` | exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现；`9a20676…` 已 push、PR #3、实现 run `33164609388` 4/4，clean capacity/9/9 acceptance 全绿；evidence-doc commit `ec29596…` 已 push且精确 SHA run `33165775037` 4/4 | +| P2-07 验证与运维 | `ready/next` | P2-01/P2-06 已提供 capacity/PG/acceptance 基线，但 P2-07 本身尚未实现；下一步开展数据库+keyring backup/restore、Redis 重建、告警响应与完整失败矩阵 | ### 验收标准 @@ -215,12 +215,12 @@ - `delivered`：真实 PostgreSQL 并发领取只有一个有效 lease；自然过期后由递增 fencing token 接管，陈旧 owner 写入被拒绝。 - `delivered/partial`：pending/running 取消有真实 Compose 证据；有限重试、超时和 dead-letter 有自动化状态机/Runner 证据，但尚未把所有失败组合都纳入完整生产式故障演练。 - `delivered`：历史 `0004` / 12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 -- `pending_on_docs_ci`：P2-06 的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已在 SHA `9a20676dcf545040782f04c166205d0043345753` 实现；本地 lint/test/integration/rules 与修复后 76-file 技术/安全终审全绿，实现已 push 到 PR #3，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）9/9、Worker `2/2/2/0/0`、cleanup C/V/N empty；clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）记录 QPS `7.267474/12.962228/9.333604`、wall `8.255963/4.628834/6.428385s`、18/270/270/271/1230、0 question error/drift/duplicate/PEL/lag、expected=2/shortfall=0 与 cleanup C/V/N/image=0、image `1/1/0/0`。两者均为 clean-SHA Mock-only evidence，不是 SLO；此前 dirty evidence 保留为历史。只剩本 evidence-doc commit 的精确 SHA CI，因此 P2-06 尚不能 `completed`。 +- `completed`：P2-06 的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已在 SHA `9a20676dcf545040782f04c166205d0043345753` 实现；本地 lint/test/integration/rules 与修复后 76-file 技术/安全终审全绿，实现已 push 到 PR #3，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）9/9、Worker `2/2/2/0/0`、cleanup C/V/N empty；clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）记录 QPS `7.267474/12.962228/9.333604`、wall `8.255963/4.628834/6.428385s`、18/270/270/271/1230、0 question error/drift/duplicate/PEL/lag、expected=2/shortfall=0 与 cleanup C/V/N/image=0、image `1/1/0/0`。两者均为 clean-SHA Mock-only evidence，不是 SLO；此前 dirty evidence 保留为历史。Evidence-doc commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 成功，P2-06 仓库级收尾完成。 - `delivered`：`P2-local-control-plane-v2` 在 clean SHA `b6a35fe…` 完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 和 `qualified` 容量模型；只限定记录的 Mock-only 单机拓扑。 -- `not_started`：P2-07 backup/restore、Redis 重建、告警响应与完整恢复矩阵尚未开始；三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 +- `ready/next`：P2-07 尚未实现；下一步开展 backup/restore、Redis 重建、告警响应与完整恢复矩阵。三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 - `delivered`：治理实现 SHA `665244e…` 已 push；本地 enhanced capacity、9/9 acceptance 与远程 run `33099260233` 4/4 均通过。 - `delivered`：P2-01 v2 实现 SHA `b6a35fe…` 已 push；本地 aggregate SHA-256 `a76d167b…d0d9` 为 23/23，实现 run `33146681285` 4/4 成功；证据文档 commit `875f13a…` 已 push，收尾 run `33150080341` 4/4 成功。 -- `delivered`：P2-06 实现 SHA `9a20676…` 已 push 至 PR #3，实现 run `33164609388` 4/4 成功，绑定同一 SHA 的 clean capacity 与 9/9 acceptance 已通过；当前仅 evidence closeout 文档 CI 待完成。 +- `delivered`：P2-06 实现 SHA `9a20676…` 已 push 至 PR #3，实现 run `33164609388` 4/4 成功，绑定同一 SHA 的 clean capacity 与 9/9 acceptance 已通过；evidence closeout 文档 SHA `ec29596…` 的 run `33165775037` 也已 4/4 成功，仓库级收尾完成。 - `delivered`：既有 API、离线 Mock Smoke 和 `llmbenchlab-protocol-v1` 评分/聚合回归继续通过，没有调用真实 Provider。 ### 风险 @@ -235,13 +235,13 @@ - 已交付基础：PostgreSQL/SQLite migration、Redis/Worker/lease/fencing/recovery、六服务 Compose 与历史故障证据。 - 已交付候选切片：`0004`、历史 12 表 importer、四层治理/ledger/backpressure/fairness、typed audit/history/Provider/credential evidence、UI 状态、enhanced capacity/PG tests 与运维文档。 -- 已完成实现门禁、待 evidence-doc CI 的切片：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator 和全日志源治理。 +- 已完成 P2-06：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator、全日志源治理，以及实现/evidence-doc 两个精确 SHA 的远程 CI。 - 治理候选门禁已完成；P2-01 的 v2 多轮资格、实现 commit/push、实现 SHA 4/4 CI 与证据文档精确 SHA 4/4 CI 也已完成，P2-01 仓库级收尾完成。 -- 阶段正式剩余：先完成 P2-06 evidence closeout 文档 commit 的 push 与精确 SHA CI；随后再启动 `not_started` 的 P2-07 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 +- 阶段正式剩余：启动 `ready/next`、尚未实现的 P2-07 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 ### 状态 -`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 为 `pending_on_docs_ci`，实现与 clean evidence 门禁已完成但本 evidence-doc commit 尚无精确 SHA CI；P2-07 为 `not_started`。不得把 P2-06、Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 +`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 的实现与 evidence-doc 精确 SHA CI 均已完成，状态为 `completed`；P2-07 为 `ready/next`，尚未实现。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 ## 6. Phase 3：标准 Benchmark 与代码评测 diff --git a/docs/TESTING.md b/docs/TESTING.md index 84f1f40..d6f6685 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -27,7 +27,7 @@ LLMBenchLab 的默认验收路径必须完全离线、可重复且不产生模 | 构建检查 | 确认生产前端可编译打包 | `npm run build` | 安装完成后不需要 | | 配置检查 | 校验 Compose 插值和服务定义 | `docker compose config` | 不启动 Provider | -单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有已交付自动化；P2-06 又加入 Worker progress、exporter/规则、retention 与全日志源回归。合并定向、lint、全量 test、smoke、真实 integration、隔离 migration、build、Compose config、Prometheus 规则解析与提交前 dirty capacity/9/9 acceptance 已全绿；structured-extra High 与 Worker `__main__` logger Medium 修复后，76-file staged 技术/安全终审为 0 Blocker/High/Medium。实现已在 clean commit `9a20676dcf545040782f04c166205d0043345753` 上重跑 capacity/acceptance 并绑定 `dirty=false` evidence，该精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 必需 job 成功。P2-06 仓库收尾仍为 `pending_on_docs_ci`；Phase 2 保持 `in_progress`，P2-07 保持 `not_started`。 +单元测试定位纯逻辑错误；真实 PostgreSQL/Redis 集成测试验证方言和队列语义；Smoke 证明最小离线链路；Compose 验收覆盖真实独立进程故障；capacity harness 记录单轮基线；SLO wrapper 在固定 clean SHA 上完成多轮资格。这些层级不能互相替代。四层治理、逐 attempt ledger、背压/公平调度、typed audit/history 和 Provider metadata 已有已交付自动化；P2-06 又加入 Worker progress、exporter/规则、retention 与全日志源回归。合并定向、lint、全量 test、smoke、真实 integration、隔离 migration、build、Compose config、Prometheus 规则解析与提交前 dirty capacity/9/9 acceptance 已全绿；structured-extra High 与 Worker `__main__` logger Medium 修复后，76-file staged 技术/安全终审为 0 Blocker/High/Medium。实现已在 clean commit `9a20676dcf545040782f04c166205d0043345753` 上重跑 capacity/acceptance 并绑定 `dirty=false` evidence，该精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 必需 job 成功。证据文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [GitHub Actions run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确绑定该 SHA 并 4/4 必需 job 成功，因此 P2-06 repository closeout 已完成。Phase 2 保持 `in_progress`；P2-07 已成为下一可执行切片，但尚未实施。 ## 3. 环境准备 @@ -352,7 +352,7 @@ Smoke Test 证明 API 与 Worker 责任边界以及数据库驱动的最小离 真实模型验收应在隔离的本地数据库上先运行 `--limit`，人工核对模型发现、付费 canary、请求上界、逐题错误、Provider 账单和报告三文件，再决定是否 `--full`。这项手工操作不得写入自动化测试结果或 CI 通过数；如果本次没有真实 API URL/Key，就应明确记录“未运行”，不能用 MockTransport 结果替代“真实 Provider 已验证”。 -当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider metadata、Worker progress、固定 exporter/八规则和 canonical audit retention；合并定向套件、提交前 dirty capacity/9/9 acceptance 与实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 clean capacity/9/9 acceptance 全绿，真实 PostgreSQL/Redis integration 为 `33 passed, 0 skipped`，同一实现 SHA 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。已知观测缺口还包括 `resume` canary 未追加独立 audit event、仓库不部署 sender/trace/WORM；不能把这些边界写成 Provider exactly-once、生产监控平台或 Phase 2 完成。P2-06 仓库收尾的唯一剩余门禁是证据文档 commit/push 及其精确 SHA CI，状态为 `pending_on_docs_ci`；P2-07 尚未开始。 +当前自动化已验证 typed lifecycle/credential audit、history counters/latency、逐题 Provider metadata、Worker progress、固定 exporter/八规则和 canonical audit retention；合并定向套件、提交前 dirty capacity/9/9 acceptance 与实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 clean capacity/9/9 acceptance 全绿，真实 PostgreSQL/Redis integration 为 `33 passed, 0 skipped`，同一实现 SHA 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。已知观测缺口还包括 `resume` canary 未追加独立 audit event、仓库不部署 sender/trace/WORM；不能把这些边界写成 Provider exactly-once、生产监控平台或 Phase 2 完成。证据文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 精确绑定该 SHA 并 4/4 成功，P2-06 repository closeout 已完成；Phase 2 仍为 `in_progress`，P2-07 是下一可执行切片且尚未实施。 ## 9. 前端测试要求 @@ -385,9 +385,9 @@ GitHub Actions 对 `main` push 和 Pull Request 触发四类 job： | `full-stack-reliability` | `python3 scripts/phase2_acceptance.py` 的隔离 Compose 九场景 | 唯一项目/卷、随机 loopback 端口、Mock-only；总是上传已脱敏 evidence，脚本总是精确清理 | | `frontend` | ESLint、Vitest 组件测试、production build（`tsc -b` + Vite） | `npm ci` 锁定依赖；fetch/Recharts stub；`VITE_API_BASE_URL=/api/v1` | -CI 不配置 Provider Key、不调用真实模型，也不在线下载 MMLU-Pro/GPQA。PostgreSQL/Redis 是测试依赖，不是 Provider 网络；标准数据转换只使用 fixture fetcher。`P2-local-control-plane-v2` 的 validator、统计、四-cell 编排失败路径、ledger projection 和 exact-project cleanup 合同可以进入普通自动化，但 GitHub-hosted runner 不运行 `make phase2-slo` 的绝对吞吐/延迟门禁：共享 runner 的 CPU、内存和 Docker 调度不是稳定性能实验室。P2-01 正式实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功；它是同一 SHA 的正确性门禁，不替代本机 1+5。P2-06 实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 也已 4/4 成功。所有必需 job 通过后才能合并；跳过用例、降低断言或使用 `continue-on-error` 都不算修复。具体分支和 Review 门槛见 [GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md)。 +CI 不配置 Provider Key、不调用真实模型，也不在线下载 MMLU-Pro/GPQA。PostgreSQL/Redis 是测试依赖，不是 Provider 网络；标准数据转换只使用 fixture fetcher。`P2-local-control-plane-v2` 的 validator、统计、四-cell 编排失败路径、ledger projection 和 exact-project cleanup 合同可以进入普通自动化，但 GitHub-hosted runner 不运行 `make phase2-slo` 的绝对吞吐/延迟门禁：共享 runner 的 CPU、内存和 Docker 调度不是稳定性能实验室。P2-01 正式实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功；它是同一 SHA 的正确性门禁，不替代本机 1+5。P2-06 实现 SHA `9a20676dcf545040782f04c166205d0043345753` 的 [run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 与证据文档 SHA `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 均已 4/4 成功。所有必需 job 通过后才能合并；跳过用例、降低断言或使用 `continue-on-error` 都不算修复。具体分支和 Review 门槛见 [GITHUB_WORKFLOW.md](GITHUB_WORKFLOW.md)。 -P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史过程记录，仓库收尾以 `9a20676dcf545040782f04c166205d0043345753` 绑定的 clean evidence 和精确 SHA CI 为准；证据文档自身的 commit/push/CI 未完成前，状态必须保持 `pending_on_docs_ci`： +P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史过程记录，仓库收尾以 `9a20676dcf545040782f04c166205d0043345753` 绑定的 clean evidence 和实现精确 SHA CI、以及证据文档 SHA `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的精确 SHA CI 为准；两层门禁均已完成： | 验证 | 当前实际结果 | | --- | --- | @@ -403,13 +403,13 @@ P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史 | Ruff scripts 补充检查 | 对 `scripts/` 的过宽默认 Ruff 命令报告 93 条既有 modernization 告警；按本任务合同使用 `--select E,F,I` 后通过，未把既有告警冒充本次失败 | | 提交前 dirty Compose acceptance（历史） | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和容器/卷/网络空清理均通过 | | 提交前 dirty Compose capacity（历史） | evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json`，SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`，`dirty=true`；1W/2W/burst wall `8.257520/4.640051/7.161722s`、`7.266104/12.930892/8.377873 q/s`，最终 18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1229 audit，0 question error/drift/duplicate/PEL/lag，expected Worker=2，cleanup C/V/N/image 全零且 image `1/1/0/0`；offline Mock、非 SLO | -| 实现提交 / PR | clean commit `9a20676dcf545040782f04c166205d0043345753` 已普通 push 至 `codex/complete-evaluation-workflow`；[PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 head 精确指向该 SHA | +| 实现提交 | clean commit `9a20676dcf545040782f04c166205d0043345753` 已普通 push 至 `codex/complete-evaluation-workflow` 并通过 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 进行审查 | | Clean-SHA Compose acceptance | 9/9；evidence `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`；`repository.commit=9a20676dcf545040782f04c166205d0043345753`、`dirty=false`、`status=passed`；Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，application `0005` populated refusal、isolated populated `0004` refusal、两层空库往返和 cleanup C/V/N empty 均通过 | | Clean-SHA Compose capacity | evidence `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；`repository.commit=9a20676dcf545040782f04c166205d0043345753`、`dirty=false`、`status=passed`；1W/2W/burst wall `8.255963/4.628834/6.428385s`、`7.267474/12.962228/9.333604 q/s`，18 Runs/270 Responses/270 question executions/271 reservations（270 actual + 1 conservative）/1230 audit，0 question error/drift/duplicate/PEL/lag；lease-owner SIGKILL 恢复场景有 1 个预期 failed attempt，expected Worker=2，cleanup C/V/N/image=0 且 image `1/1/0/0`；offline Mock、非 SLO | | 实现提交前 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file index 为 0 Blocker/High/Medium；hydration/import integrity 修复目标集 `67 passed` | | 实现精确 SHA 远程 CI | [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对 `9a20676dcf545040782f04c166205d0043345753` 的 backend、backend-integration、full-stack reliability 与 frontend 4/4 必需 job 全部成功 | -| 仍待门禁 | 形成证据文档 commit，普通 push 并等待该文档精确 SHA 的四个必需 CI job 全部成功 | -| 状态边界 | P2-06 repository closeout=`pending_on_docs_ci`；Phase 2=`in_progress`；P2-07=`not_started` | +| 证据文档提交 / 精确 SHA 远程 CI | commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已普通 push；[GitHub Actions run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 精确绑定该 SHA，backend、backend-integration、full-stack reliability 与 frontend 4/4 必需 job 全部成功 | +| 状态边界 | P2-06 repository closeout=`completed`；Phase 2=`in_progress`；P2-07 为下一可执行切片，尚未实施 | 下表是 2026-08-25 Phase 2 可靠性切片的历史基线，不是本次标准数据/真实评测 CLI 提交的通过声明。本次实际数量必须在当前工作日志和精确 commit 的 CI 中重新记录，不能沿用这些数字： diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index e9426e2..b580ca1 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -28,7 +28,7 @@ - 每个 Provider HTTP attempt 的 reserve→send-started→actual/conservative settlement 或 confirmed pre-send release never-delete ledger；materialized scope/bucket counter 仅作投影，任何高/低漂移 fail closed。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model due ordering。 - typed audit、分页 Run audit、task history counters、基于 Run 数据库时间戳的 queue/execution/end-to-end latency、严格规范化 Provider metadata 和非秘密 credential audit。 -- 固定低基数 Prometheus text exporter、八条仓库内告警规则/Runbook、DB-time Worker generation/progress 聚合、canonical audit archive/离线 verify/reconcile/restore/delete 和全日志源治理；这些 P2-06 功能已进入 clean implementation SHA 并通过 clean Compose 与远程实现门禁，仓库级收尾只待证据文档提交自身的精确 SHA CI。 +- 固定低基数 Prometheus text exporter、八条仓库内告警规则/Runbook、DB-time Worker generation/progress 聚合、canonical audit archive/离线 verify/reconcile/restore/delete 和全日志源治理；这些 P2-06 功能、clean Compose、实现 SHA 与证据文档 SHA 远程门禁均已完成。 - Run Detail 展示 managed/delayed/exhausted、治理原因和明确 UTC not-before；旧 Run 与可信本地 CLI 明确为 `legacy_unmanaged`。 - 真实 PostgreSQL 竞争测试及 Mock-only enhanced capacity/acceptance；精确实现 SHA 的完整 capacity、9/9 acceptance、`P2-local-control-plane-v2` 多轮单机资格与远程 4/4 CI 已通过。 @@ -65,8 +65,8 @@ | P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；`9a20676…` 增加 generation 级 DB-time scan/claim/lease-heartbeat/progress 与 stale 聚合，dependency probe 仍只表示 capability | | P2-04 生命周期可靠性 | `foundation_delivered` | retry/backoff、取消、恢复、dead-letter、Response 幂等和三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；Provider 外部副作用仍为 at-least-once | | P2-05 并发治理 | `slice_delivered` | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序、counter 重算 fail-closed 与 ADR-0011 已实现；精确 SHA 的真实 PG/capacity/acceptance/CI 候选门禁已通过 | -| P2-06 可观测性 | `pending_on_docs_ci` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已进入 clean commit `9a20676…`；clean capacity/9/9 acceptance 与该实现 SHA 的 CI 4/4 已通过，只待证据文档提交自身的精确 SHA CI | -| P2-07 验证与运维 | `not_started` | 既有 enhanced capacity/PG tests、正式 v2 单机资格与运维文档是共享前置证据，不代表本切片已启动；PostgreSQL+keyring backup/restore、Redis 重建、完整失败矩阵与告警响应尚未实施 | +| P2-06 可观测性 | `completed` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已进入 clean commit `9a20676…`；clean capacity/9/9 acceptance、实现 CI 与 evidence-doc commit `ec29596…` 的 CI 4/4 均通过 | +| P2-07 验证与运维 | `ready/next` | 既有 enhanced capacity/PG tests、正式 v2 单机资格与运维文档是共享前置证据；本切片尚未实现，下一步须先新建独立计划/日志/必要 ADR，再实施 PostgreSQL+keyring backup/restore、Redis 重建、完整失败矩阵与告警响应 | `slice_delivered` 表示该垂直切片及其候选门禁已交付，不表示整个阶段完成。Phase 2 必须保持 `in_progress`。 @@ -80,7 +80,7 @@ - [x] **crash seam 验收**：`reserved`→send-start、`send_started`→settlement、Response commit→最终恢复三条 deterministic DB seam injection 在完整 Compose acceptance 通过；它们不冒充精确时刻 `SIGKILL`。 - [x] **远程实现门禁**：GitHub Actions run [`33099260233`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33099260233) 对精确实现 SHA 4/4 成功。 - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 -- [ ] **P2-06 仓库级闭环待证据文档 CI**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [GitHub Actions run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 已通过；只待本次证据文档提交、push 与该文档精确 SHA CI 全绿。 +- [x] **P2-06 仓库级闭环完成**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确 4/4 通过。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -109,7 +109,7 @@ | P2-06 implementation remote gate | PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) | 精确 `9a20676dcf545040782f04c166205d0043345753`，四个必需 job 全 success | | P2-06 补充 Ruff | 过宽 scripts 命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | 保留首次结果，不扩大本切片清理范围 | | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | -| P2-06 剩余门禁 | 本次证据文档 commit、push 与该文档精确 SHA CI | 实现/clean-SHA Compose/implementation CI 已完成；完成文档门禁前状态为 `pending_on_docs_ci` | +| P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 @@ -128,15 +128,15 @@ | 限额并发突破 | canonical scope、固定锁序、DB transaction、ledger 重算、真实 PG integration 与 v2 多轮资格 | 超出固定单机 profile 时重新测量并持续回归 | | Provider 调用/费用重复 | send-start marker、保守结算、本地幂等及三条 crash seam acceptance | 外部 exactly-once 不可承诺 | | 长 Run 饥饿 | finite quantum、due ordering、dispatch/failure 分离及 v2 每轮公平性硬门禁 | 更大规模或不同 Worker 拓扑需重新建模 | -| Worker 停滞不可见 | DB-time progress/liveness 聚合、exporter 与 `WorkerStalled` rule；dependency probe 保持 capability-only | 完成 P2-06 文档门禁，并在 P2-07 演练告警处置/扩缩 | -| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination、canonical archive/verify/精确 delete/restore | 完成 P2-06 文档门禁；P2-07 验证异地存储与整库恢复边界 | +| Worker 停滞不可见 | DB-time progress/liveness 聚合、exporter 与 `WorkerStalled` rule；dependency probe 保持 capability-only | 在 P2-07 演练告警处置/扩缩 | +| 审计增长/泄密 | 固定 allowlist、无正文/URL/Key、pagination、canonical archive/verify/精确 delete/restore | P2-07 验证异地存储与整库恢复边界 | | 容量结论过度外推 | Mock-only、环境/config/evidence 指纹、1+5 多轮和明确支持 profile | 不冒充 Provider/生产 SLA；环境或 profile 变化必须重新资格 | | 灾难恢复失败 | 13 表 importer、迁移 guard、audit archive 自身 restore、独立 keyring 边界 | PostgreSQL backup/restore、keyring 配对、Redis 重建与完整恢复演练 | ## 交付物与下一任务 -已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 已包含 `0005` / 13 表 importer、Worker DB-time progress、exporter/八规则、audit retention CLI 与日志治理，并通过 clean capacity/9/9 acceptance 和远程 4/4 CI；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 完成本次证据文档提交自身的精确 SHA CI，再独立启动 P2-07 backup/restore 和剩余运维演练。 +已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 与 evidence-doc SHA `ec29596…` 已分别通过精确 4/4 CI，仓库级闭环完成；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 新建 P2-07 独立计划/日志/必要 ADR，再实施 backup/restore 和剩余运维演练。 ## 状态 -`in_progress`。P2-01 已完成，P2-05 已交付；P2-06 实现与 clean-SHA 远程门禁已通过、仓库级状态为 `pending_on_docs_ci`，P2-07 尚未启动。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +`in_progress`。P2-01 与 P2-06 已完成，P2-05 已交付；P2-07 为 `ready/next` 但尚未实现，数据库+keyring backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 diff --git a/docs/plans/2026-08-28-phase-2-observability-retention.md b/docs/plans/2026-08-28-phase-2-observability-retention.md index b0248a0..607e1fc 100644 --- a/docs/plans/2026-08-28-phase-2-observability-retention.md +++ b/docs/plans/2026-08-28-phase-2-observability-retention.md @@ -1,7 +1,7 @@ # Phase 2 可观测性与审计保留执行计划 - Owner: Codex -- Status: active (`pending_on_docs_ci`) +- Status: completed - Created: 2026-08-28 - Updated: 2026-08-28 - Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) @@ -10,7 +10,7 @@ ## Context -Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audit/history 和单机 Mock 控制面资格。本计划开始时，数据库 API 能返回任务 gauges、typed event counters 和 Run latency，但没有稳定低基数 exporter/规则、Worker 主循环进展或可执行 audit retention。P2-06 实现已进入 clean commit `9a20676dcf545040782f04c166205d0043345753`，clean-SHA capacity/9/9 acceptance 与该实现 SHA 的远程 CI 4/4 均通过；计划仍保持 active，因为本次证据文档提交及其自身精确 SHA CI 尚未完成。 +Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audit/history 和单机 Mock 控制面资格。本计划开始时，数据库 API 能返回任务 gauges、typed event counters 和 Run latency，但没有稳定低基数 exporter/规则、Worker 主循环进展或可执行 audit retention。P2-06 实现已进入 clean commit `9a20676dcf545040782f04c166205d0043345753`，clean-SHA capacity/9/9 acceptance 与实现 CI 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的自身 CI 也 4/4 通过，因此本计划完成。 ## Objective @@ -69,9 +69,9 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi - Result: 相关代码/Compose/env 及主要协议、API、安全、架构、部署、运维和状态文档已进入当前 diff；状态、测试、计划和工作日志已同步。 - Validation: 本轮九份状态文档相对链接检查与 `git diff --check` 通过；最终全仓规则/秘密审计属于步骤 6。 - Logging: 生产 logger 消息必须为无格式参数的字面量；第三方动态消息统一固定化，Uvicorn raw access log 禁用。 -6. [in_progress] 执行完整验证、终审和远程门禁。 +6. [completed] 执行完整验证、终审和远程门禁。 - Files/modules: 全部本切片文件与状态文档。 - - Validation: 定向+全量+lint+smoke+双方言 migration/PG integration+Compose/规则/secret/diff、clean-SHA capacity/acceptance 与实现 SHA CI 4/4 已通过；剩余门禁仅为证据文档提交、push 和该文档 SHA 自身 CI 4/4。 + - Validation: 定向+全量+lint+smoke+双方言 migration/PG integration+Compose/规则/secret/diff、clean-SHA capacity/acceptance、实现 SHA CI 与 evidence-doc SHA CI 均已通过。 ## Risks @@ -106,7 +106,7 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi | Implementation staged 技术/安全终审 | Blocker/High/Medium 终审 + 受影响回归 | 无未解决 B/H/M | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | | Docs/security review | Markdown 链接、secret/diff scan | 无坏链接、秘密或虚假状态 | 状态文档相对链接、whitespace diff、staged secret/path/blob 扫描均通过 | | Implementation remote gate | push 后 GitHub Actions exact SHA | 四个必需 job 全 success | PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)；[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对 `9a20676dcf545040782f04c166205d0043345753` 4/4 success | -| Evidence-doc remote gate | 本次文档 commit 的 GitHub Actions exact SHA | 四个必需 job 全 success | 待本次文档提交、push 后执行；完成前 P2-06 保持 `pending_on_docs_ci` | +| Evidence-doc remote gate | 本次文档 commit 的 GitHub Actions exact SHA | 四个必需 job 全 success | commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`；[run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 success | ## Rollback @@ -118,7 +118,7 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi - [x] API / Schema 已进入当前 P2-06 diff - [x] Architecture / Security / ADR-0015 已进入当前 P2-06 diff - [x] Testing / Deployment / Operations / alert Runbook 已进入当前 P2-06 diff -- [x] CHANGELOG、PROJECT_STATUS、Roadmap、Phase 2、NEXT_TASK、工作日志已同步；最终门禁结果仍须在步骤 6 回填 +- [x] CHANGELOG、PROJECT_STATUS、Roadmap、Phase 2、NEXT_TASK、工作日志已同步；步骤 6 的实现与 evidence-doc 精确 SHA 门禁结果均已回填 ## Completion evidence @@ -126,8 +126,9 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi - Commands run: `make lint` 全绿（Ruff 152 files、ESLint、TypeScript typecheck）；`make test` 后端 `916 passed, 33 skipped`、前端 `38 passed`；`make smoke` 为 `1 passed, 7 deselected`；临时 PostgreSQL 16/Redis 7 migration/check 与 integration `33 passed, 0 skipped`；临时 SQLite head→0001→head/check、正确目录 frontend build、`docker compose config --quiet`、Prometheus `v3.5.0` 八规则校验、clean-SHA capacity/9/9 acceptance 成功。 - Acceptance evidence: implementation commit `9a20676dcf545040782f04c166205d0043345753` 的 clean acceptance artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`；clean capacity artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`。两者 `dirty=false`、status passed；acceptance cleanup C/V/N empty，capacity cleanup C/V/N/image empty。Capacity 是 offline Mock 非 SLO。 - Remote evidence: implementation commit 已 push 到 `origin/codex/complete-evaluation-workflow`；PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3) 的 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 对该精确 SHA 4/4 success。 -- Not run: 本次证据文档提交尚未形成，因此尚无该文档精确 SHA 的 CI；默认用户 SQLite 未迁移到 head，按保护原则保留原状。 -- Known issues: P2-07 仍为下一独立切片；Phase 2 保持 `in_progress`。 +- Evidence-doc closeout: commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push；[run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 精确绑定该 SHA 并 4/4 success。 +- Not run: 默认用户 SQLite 未迁移到 head，按保护原则保留原状；这不影响隔离双方言 migration 与 CI 门禁。 +- Known issues: P2-07 仍为下一独立切片且尚未启动；Phase 2 保持 `in_progress`。 ## Decision and discovery log @@ -144,3 +145,4 @@ Phase 2 已交付 durable Worker、租约/fencing、数据库治理、typed audi | 2026-08-28 | gate | Prometheus v3.5.0 八规则、dirty capacity 与 9/9 acceptance 通过；structured-extra High 与 Worker `__main__` logger Medium 修复后，最新 staged 技术/安全终审为 0 Blocker/High/Medium。 | acceptance `11554c25ec2d` / `d5f058…9fde`；capacity `c6de062ab77e` / `4aeb827…8453`；clean-SHA 仍须重跑。 | | 2026-08-28 | gate | clean implementation commit `9a20676…` 的 capacity/9/9 acceptance 与远程 run `33164609388` 4/4 通过。 | clean artifacts 为 acceptance `92e173eeee28` / `e4ffb866…0fc47`、capacity `ca5673061b0f` / `2382f913…f5039`；不复用 dirty evidence冒充 clean gate。 | | 2026-08-28 | status | P2-06 实现与 clean-SHA/implementation-CI 门禁已完成；证据文档提交及其自身 exact-SHA CI 尚未完成。 | P2-06=`pending_on_docs_ci`、Phase 2=`in_progress`、P2-07=`not_started`；先完成步骤 6 再启动 P2-07。 | +| 2026-08-28 | closeout | evidence-doc commit `ec29596…` 已 push，run `33165775037` 对该精确 SHA 4/4 success。 | P2-06=`completed`、本计划=`completed`；Phase 2 因 P2-07 仍为 `in_progress`。 | diff --git a/docs/worklogs/2026-08-28-phase-2-observability-retention.md b/docs/worklogs/2026-08-28-phase-2-observability-retention.md index 494a831..edff672 100644 --- a/docs/worklogs/2026-08-28-phase-2-observability-retention.md +++ b/docs/worklogs/2026-08-28-phase-2-observability-retention.md @@ -10,7 +10,7 @@ - 关联计划：[Phase 2 可观测性与审计保留执行计划](../plans/2026-08-28-phase-2-observability-retention.md) - 决策基础：[ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](../decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) - 本轮决策：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) -- 当前状态：`pending_on_docs_ci`；P2-06 implementation/clean-SHA/remote gate 已通过，证据文档精确 SHA CI 尚待；Phase 2 仍为 `in_progress` +- 当前状态：`completed`；P2-06 implementation、clean-SHA、evidence-doc 与各自精确 SHA CI 均通过；Phase 2 仍为 `in_progress` ## 目标与背景 @@ -117,7 +117,7 @@ P2-01 已在精确提交与远程 CI 上闭环，现有系统也已经提供 Pos ### 状态边界 -- 以上功能已进入 clean implementation commit `9a20676dcf545040782f04c166205d0043345753`。lint/test/smoke/integration/migration/build/config/rules、clean-SHA capacity/9/9 acceptance、push 与该实现 SHA 的 GitHub Actions 4/4 均已通过。staged security review 发现的 structured logging High 与 `python -m app.worker` logger Medium 已修复，最终技术/安全终审为 0 Blocker/High/Medium。当前只待本次证据文档提交及其自身精确 SHA CI，因此 P2-06 仓库级状态为 `pending_on_docs_ci`。 +- 以上功能已进入 clean implementation commit `9a20676dcf545040782f04c166205d0043345753`。lint/test/smoke/integration/migration/build/config/rules、clean-SHA capacity/9/9 acceptance、push 与该实现 SHA 的 GitHub Actions 4/4 均已通过。staged security review 发现的 structured logging High 与 `python -m app.worker` logger Medium 已修复，最终技术/安全终审为 0 Blocker/High/Medium。Evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 及其自身精确 SHA CI 也已通过，P2-06 仓库级状态为 `completed`。 - P2-06 archive restore 只恢复 typed audit archive 自身。PostgreSQL+keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置与剩余故障矩阵仍属于 P2-07；Phase 2 继续 `in_progress`。 ## 实际命令与结果 @@ -148,14 +148,15 @@ P2-01 已在精确提交与远程 CI 上闭环，现有系统也已经提供 Pos | `make phase2-acceptance`（clean SHA） | implementation commit `9a20676dcf545040782f04c166205d0043345753`，artifact `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`，SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，`dirty=false`、9/9；Worker `2/2/2/0/0`，两级 populated refusal、两层空库往返、queue 0/0 与 cleanup C/V/N empty；脚本不承诺 build image cleanup | | `make phase2-capacity`（clean SHA） | 同一 implementation commit，artifact `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`，SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`，`dirty=false`；1W/2W/burst wall `8.255963/4.628834/6.428385s`、`7.267474/12.962228/9.333604 q/s`；18/270/270/271/1230，0 question error/drift/duplicate/PEL/lag，expected Worker=2、stalled/shortfall=0，故障恢复后的瞬时 registered/live=3；cleanup C/V/N/image=0 且 image `1/1/0/0`；offline Mock、非 SLO | | implementation commit / push / exact-SHA CI | commit `9a20676dcf545040782f04c166205d0043345753` 已普通 push 至 `origin/codex/complete-evaluation-workflow`；PR [#3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)；[run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 精确绑定该 SHA，Frontend、Compose reliability、Backend、PG/Redis integration 四个 job 全 success | +| evidence-doc commit / push / exact-SHA CI | commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已普通 push；[run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 精确绑定该 SHA，四个必需 job 全 success | | 先前 code review 与 hydration/import integrity 回归 | 当时结论为 0 Blocker/High/Medium、目标集 `67 passed`；其后 staged security review 发现 structured logging High 与 Worker entrypoint Medium，均已修复并重新复核 | | Implementation staged 技术/安全终审 | 76-file implementation index 为 0 Blocker/High/Medium；独立 logging/source/queue `19 passed`，archive/retention/CLI/exporter/Docker-script `167 passed`；secret/path/blob/category 扫描无真实凭据或禁止产物 | -| README、TESTING、CHANGELOG、PROJECT_STATUS、ROADMAP、Phase 2、NEXT_TASK、计划/工作日志状态同步 | 已把实现事实与 pending-gate/P2-07 边界写入当前 diff；本轮链接/diff 检查通过 | +| README、TESTING、CHANGELOG、PROJECT_STATUS、ROADMAP、Phase 2、NEXT_TASK、计划/工作日志状态同步 | 已回填 implementation 与 evidence-doc 精确 SHA 门禁，并把 P2-06 completed / P2-07 ready-next 边界写入状态文档；本轮链接/diff 检查通过 | | 九份状态文档相对链接检查 | 全部目标存在 | | 七份 tracked 状态文档 `git diff --check` + 新 plan/worklog 的 `git diff --no-index --check` | 无 whitespace error；no-index 命令因文件整体为新增返回差异状态 1，且没有 `--check` 诊断 | ## 已知问题与下一步 - P2-06 三条实现线、structured logging High 与 Worker entrypoint Medium 修复已落地；lint、全量 test、smoke、frontend build、临时 SQLite/真实 PG migration、真实 PG/Redis integration、Compose config、八规则 promtool、clean-SHA capacity/9/9 acceptance、implementation push 与精确 SHA 4/4 CI 均已通过。 -- 当前只需完成本次 evidence-doc commit、普通 push 与该文档精确 SHA 四个 CI job；绿色前 P2-06 为 `pending_on_docs_ci`，不得写成 completed。绿色后另做状态收尾提交并等待其自身 CI。 -- P2-07 未开始，仍缺数据库+keyring 配对 backup/restore、Redis 重建、告警响应与完整恢复矩阵；Phase 2 继续 `in_progress`。 +- P2-06 implementation、clean-SHA evidence、证据文档提交及对应精确 SHA CI 均已完成；本工作日志状态为 `completed`。 +- P2-07 尚未开始，仍缺数据库+keyring 配对 backup/restore、Redis 重建、告警响应与完整恢复矩阵；下一步须新建独立计划/日志/必要 ADR，Phase 2 继续 `in_progress`。 From 6df84d9bb26ee81c2f91ed9772dd6f45e9e17741 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Fri, 28 Aug 2026 19:40:35 +0800 Subject: [PATCH 04/12] docs: plan phase 2 recovery operations --- CHANGELOG.md | 5 +- README.md | 4 +- docs/NEXT_TASK.md | 23 +- docs/PROJECT_STATUS.md | 8 +- docs/ROADMAP.md | 10 +- ...esql-keyring-recovery-and-redis-rebuild.md | 216 ++++++++++++++++++ docs/phases/PHASE-2-RELIABILITY.md | 6 +- .../2026-08-28-phase-2-recovery-operations.md | 140 ++++++++++++ .../2026-08-28-phase-2-recovery-operations.md | 110 +++++++++ 9 files changed, 495 insertions(+), 27 deletions(-) create mode 100644 docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md create mode 100644 docs/plans/2026-08-28-phase-2-recovery-operations.md create mode 100644 docs/worklogs/2026-08-28-phase-2-recovery-operations.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a04abd1..a747c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Added +- Added the P2-07 recovery-operations kickoff package: ADR-0016, an independent execution plan, and a work log now freeze a deliberately narrow PostgreSQL 16 dump/restore verification, separate keyring pairing, Redis-as-notification rebuild, alert-drill, destructive-operation, and evidence boundary. This commit contains documentation only; P2-07 implementation and qualification have not started, Phase 2 remains `in_progress`, and no production DR/PITR/RPO/RTO/HA claim is made. - Added the completed P2-06 implementation under ADR-0015: `GET /api/v1/metrics/prometheus` renders a fixed Prometheus text `0.0.4` gauge set from one DB-time snapshot, with enum-only labels, bounded 15-minute audit and one-hour latency windows, fail-closed audit validation, and one in-flight collection per API process. Repository-owned configuration supplies exactly eight alert rules and fixed Operations runbook links; Prometheus, Alertmanager, and notification delivery are not deployed by this repository. - Added Alembic revision `20260828_0005`, the `worker_processes` generation table, bounded `(expires_at,id)` / `(occurred_at,id)` audit scan indexes, and DB-time coalesced Worker scan/claim/lease-heartbeat/progress recording. `/tasks/metrics` and the exporter expose only low-cardinality aggregates; the dependency probe remains capability-only. The stopped SQLite-to-PostgreSQL importer now copies and reconciles all 13 application tables, rejects a still-live Worker generation, and preserves stopped/stale process facts. - Added `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete`. The v1 canonical JSONL archive is bounded, exact-digest bound, atomically installed without replacement at `0600`, and independently verifiable without a database engine; delete is never implicit, restore/delete require an exact confirmed archive SHA-256, and commit-unknown outcomes require read-only reconciliation before retry. These integrity hashes are neither signatures nor WORM guarantees. @@ -107,11 +108,11 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Verification -- P2-06 implementation gates are complete. Commit [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) was pushed to `codex/complete-evaluation-workflow` and is tracked by [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3); exact-SHA GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) passed all four required jobs. Its local gates include green combined targeted suites, `make lint` across 152 Python files plus ESLint/TypeScript, backend `916 passed, 33 skipped`, frontend `38 passed`, offline Mock smoke `1 passed, 7 deselected`, PostgreSQL 16/Redis 7 migration/check and `33 passed, 0 skipped` integration, isolated SQLite migration/check, frontend build, Compose config, eight-rule Prometheus `v3.5.0` validation, and a 76-file technical/security review with 0 Blocker/High/Medium. Clean-SHA Compose acceptance passed 9/9 with `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` (SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`), Worker gauges `2/2/2/0/0`, and empty container/volume/network cleanup. Clean capacity passed with `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` (SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`): 1W/2W/burst QPS `7.267474/12.962228/9.333604`, wall `8.255963/4.628834/6.428385s`, 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit events, zero question error/drift/duplicate/PEL/lag, Worker expected 2 with shortfall 0, and empty container/volume/network/image cleanup with image counters `1/1/0/0`. The earlier dirty acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`) and dirty capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`, 1229 audit events) remain historical evidence rather than being overwritten. These are offline Mock observations, not production or real-Provider SLOs. Evidence-documentation commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) was pushed, and its exact-SHA GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) passed all four required jobs, completing P2-06. Phase 2 stays `in_progress`; P2-07 is `ready/next` but not implemented. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. +- P2-06 implementation gates are complete. Commit [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) was pushed to `codex/complete-evaluation-workflow` and is tracked by [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3); exact-SHA GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) passed all four required jobs. Its local gates include green combined targeted suites, `make lint` across 152 Python files plus ESLint/TypeScript, backend `916 passed, 33 skipped`, frontend `38 passed`, offline Mock smoke `1 passed, 7 deselected`, PostgreSQL 16/Redis 7 migration/check and `33 passed, 0 skipped` integration, isolated SQLite migration/check, frontend build, Compose config, eight-rule Prometheus `v3.5.0` validation, and a 76-file technical/security review with 0 Blocker/High/Medium. Clean-SHA Compose acceptance passed 9/9 with `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` (SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`), Worker gauges `2/2/2/0/0`, and empty container/volume/network cleanup. Clean capacity passed with `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` (SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`): 1W/2W/burst QPS `7.267474/12.962228/9.333604`, wall `8.255963/4.628834/6.428385s`, 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit events, zero question error/drift/duplicate/PEL/lag, Worker expected 2 with shortfall 0, and empty container/volume/network/image cleanup with image counters `1/1/0/0`. The earlier dirty acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-11554c25ec2d/evidence.json` (SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`) and dirty capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-c6de062ab77e/evidence.json` (SHA-256 `4aeb8271dd81e8671fc287942839f8d06862140ea9a6bf1d7ee5660265aa8453`, 1229 audit events) remain historical evidence rather than being overwritten. These are offline Mock observations, not production or real-Provider SLOs. Evidence-documentation commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) was pushed, and its exact-SHA GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) passed all four required jobs, completing P2-06. Phase 2 stays `in_progress`; P2-07 now has a `planned` work package but remains unimplemented. The default user SQLite was below head, so its direct `alembic check` failed and it was deliberately not migrated. - Recorded two corrected local gate invocations without hiding their first outcomes: the initial integration cleanup command was rejected by the local safety policy before any container started, then the explicit cleanup/run passed; an over-broad Ruff invocation over `scripts/` surfaced 93 existing modernization warnings, while the intended `--select E,F,I` correctness/import gate passed. - On the governance/audit candidate at `665244e095905083b606b8e98e946ed1a02dc0fc`, `make test` passed with `604 passed, 29 skipped` on the backend and `38 passed` on the frontend; a separate real PostgreSQL/Redis integration run passed `29/29`. That frozen implementation also passed `make lint`, offline `make smoke` (`1 passed, 7 deselected`), the frontend production build, isolated SQLite and PostgreSQL migration/check gates, `docker compose config --quiet`, targeted governance/API/Worker suites, enhanced capacity, and 9/9 Compose acceptance. - Pushed the implementation and lock-order fixes through exact SHA `665244e095905083b606b8e98e946ed1a02dc0fc`. Its enhanced capacity evidence (`40deadeb…0588`) records finite policy, exact 4×`202` + 2×`429`, cooperative/cross-Model fairness, fault recovery, 18 Runs/270 Responses/271 reservations/1229 audit events, and empty cleanup; its 9/9 acceptance evidence (`ab311665…ddec`) includes all three deterministic database seams and empty cleanup. GitHub Actions run `33099260233` passed all four required jobs. At that gate, the formal SLO/exporter/retention/backup/Worker-progress closure was still pending. -- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress`; P2-06 is `completed`, and P2-07 is `ready/next` but not implemented. +- On the later P2-01 v2 implementation SHA `b6a35fef1dd069ebb54b69955058915c722aa34d`, `make test` passed with `829 passed, 29 skipped` on the backend and `38 passed` on the frontend. GitHub Actions run `33146681285` passed all four required jobs; a fresh 1+5 formal invocation then passed 23/23 SLO checks and every child hard invariant/cleanup check, with aggregate SHA-256 `a76d167bb664e2ee3ee7514c39ac738b76cef37776d7b66e1175a8596329d0d9`. Evidence-documentation commit `875f13a253c40b7573d45c6287385e60f2bb8f04` was pushed and its exact-SHA GitHub Actions run `33150080341` passed all four required jobs, completing P2-01 repository closeout for this fixed Mock-only single-host profile. Phase 2 remains `in_progress`; P2-06 is `completed`, and P2-07 is `planned` but not implemented. - Opened PR #1 for the trusted-local evaluation/Web credential/Run UX/SSE branch and obtained all four required GitHub Actions jobs for exact SHA `ab15862eab4870dda01fb079b44b509a7d737627` in run `33078921254`; this closes the previously documented no-PR/no-CI boundary for that SHA only. - Passed 205 backend non-infrastructure tests, five real PostgreSQL/Redis integration tests, 13 frontend tests, the isolated offline Mock smoke test, Ruff/ESLint/TypeScript checks, Vite production build, and SQLite/PostgreSQL Alembic upgrade/check/downgrade/upgrade gates. - Passed an isolated default-build Compose acceptance harness in all eight scenarios: topology/readiness, protocol-v1 baseline, API restart during execution, exact lease-owner SIGKILL and natural takeover, Redis stop/start with database reconciliation, pending cancellation, running cancellation plus duplicate delivery, and PostgreSQL `head -> 0001 -> head` protocol round-trip. diff --git a/README.md b/README.md index eb352fc..6cd6a99 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - 自动化、CI、Compose 故障验收和容量演练的模型执行都只使用 Mock；根据层级使用临时 SQLite 或隔离 PostgreSQL 16/Redis 7，不访问真实模型服务，也不产生模型费用。 - Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 - P2-06 状态为 `completed`。实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已 push 到当前分支并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 [GitHub Actions run 33164609388](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 成功。绑定同一 clean SHA 的 Compose acceptance 9/9 通过，evidence 为 `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`），Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0` 且 cleanup C/V/N 全空；clean capacity evidence 为 `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`），1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 QuestionExecutions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。这是 Mock-only 单机观测，不是生产或真实 Provider SLO；此前 dirty evidence 继续保留为历史，不替代该 clean-SHA 结果。Evidence-doc commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 [run 33165775037](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 成功，完成 P2-06 仓库级收尾。 -- P2-07 状态为 `ready/next`，但尚未实现；下一步才开始数据库/keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵，因此 Phase 2 仍为 `in_progress`。Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 +- P2-07 状态为 `planned`，ADR-0016、独立计划和工作日志已建立，但功能尚未实现；后续才开始数据库/keyring 配对 backup/restore、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵，因此 Phase 2 仍为 `in_progress`。Phase 3 也只交付客观数据垂直切片，其余 Phase 3–6 能力仍未完成。 最新、可复核的完成状态与测试证据以 [`docs/PROJECT_STATUS.md`](docs/PROJECT_STATUS.md) 和 [`docs/worklogs/`](docs/worklogs/) 为准；Roadmap 中的计划能力不等于已交付能力。 @@ -396,7 +396,7 @@ uv run llmbenchlab-audit-retention delete \ | --- | --- | --- | | Phase 0 | 项目治理、需求、架构、协议 | 已完成 | | Phase 1 | FastAPI + React + SQLite 的 MVP 垂直链路 | 已完成 | -| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 已闭环；P2-06 实现与 evidence-doc 精确 SHA CI 均全绿，状态为 `completed`；P2-07 为 `ready/next`，尚未实现 | +| Phase 2 | PostgreSQL、Redis、独立 Worker、治理、恢复与可观测性 | `in_progress`：P2-01 已闭环；P2-06 实现与 evidence-doc 精确 SHA CI 均全绿，状态为 `completed`；P2-07 工作包已建立，状态为 `planned`，功能尚未实现 | | Phase 3 | 合规标准 Benchmark 与隔离代码评测 | MMLU-Pro/GPQA 客观数据切片已交付；IFEval、沙箱及其余验收未完成 | | Phase 4 | LLM Judge、人工校准与 Arena | 计划中 | | Phase 5 | Agent、工具调用与 Live Benchmark | 计划中 | diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 965ab98..5747e2b 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -1,9 +1,10 @@ -# 下一任务：启动 P2-07 恢复与运维闭环 +# 下一任务：实施 P2-07 最小恢复验证切片 -> 状态：`ready`；P2-06 已完成仓库级闭环，下一步先建立 P2-07 独立计划、工作日志与必要 ADR +> 状态：`planned`；P2-07 独立计划、工作日志与 ADR-0016 已建立，功能实现尚未开始 > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) -> 已完成计划：[Phase 2 可观测性与审计保留](plans/2026-08-28-phase-2-observability-retention.md) -> 已完成日志：[2026-08-28 P2-06 工作日志](worklogs/2026-08-28-phase-2-observability-retention.md) +> 当前计划：[Phase 2 恢复与运维闭环](plans/2026-08-28-phase-2-recovery-operations.md) +> 当前日志：[2026-08-28 P2-07 工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md) +> 当前决策：[ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) > 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) ## 当前事实 @@ -21,16 +22,16 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。默认用户 SQLite 尚未在 head且未擅自迁移。 -## 立即执行：建立 P2-07 工作包 +## 已完成：建立 P2-07 工作包 -1. 按 AGENTS/PLANS 先新建 P2-07 独立执行计划与工作日志，记录目标、非目标、验收、风险和实施步骤；不得复用 P2-06 日志冒充新任务记录。 -2. 完整勘察现有 PostgreSQL backup/restore、keyring、Redis consumer group、告警 Runbook 与 Compose/CI 边界；若改变恢复协议或安全取舍，实施前新增 ADR。 -3. 把 P2-07 拆成可独立验证的 backup/keyring、Redis rebuild、alert response 与故障矩阵步骤，先冻结恢复不变量和证据公开边界，再实施代码/脚本。 -4. 自动化只使用 Mock/Stub；任何数据库/volume/keyring 删除或替换必须使用隔离、精确目标和 fail-closed guard，不触碰默认用户数据。 +1. 已按 AGENTS/PLANS 新建独立执行计划与工作日志，记录目标、非目标、验收、风险和实施步骤。 +2. 已勘察 PostgreSQL backup/restore、keyring、Redis consumer group、告警 Runbook 与 Compose/CI 边界，并以 ADR-0016 冻结关键取舍。 +3. 已把后续工作拆成 verifier、Redis/Worker、rules/recovery harness 和最终门禁；本轮按用户要求停止，没有实施代码或脚本。 +4. 后续自动化仍只能使用 Mock/Stub；任何数据库/volume/keyring 删除或替换必须使用隔离、精确目标和 fail-closed guard，不触碰默认用户数据。 ## P2-07：下一独立切片 -P2-07 当前为 `ready`、尚未实现。新建独立计划、工作日志和必要 ADR 后，完成： +P2-07 当前为 `planned`、尚未实现。恢复实施时按当前计划从最小只读 verifier 切片开始，之后才依次完成： - PostgreSQL backup → 空目标 restore → Alembic `20260828_0005` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 - 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 @@ -54,5 +55,5 @@ P2-07 当前为 `ready`、尚未实现。新建独立计划、工作日志和必 ## 可直接复制给 Codex 的任务指令 ```text -继续执行 docs/NEXT_TASK.md。P2-06 已完成，不要重跑碰绿。现在按 AGENTS/PLANS 新建 P2-07 独立计划、工作日志和必要 ADR，先勘察并冻结 PostgreSQL+keyring 配对 backup/restore、Redis rebuild、八规则告警处置和剩余故障矩阵合同，再分步实现与验证。自动化只用 Mock/Stub，所有 destructive 操作只针对隔离、精确目标；Phase 2 在 P2-07 完成前保持 in_progress。 +继续执行 docs/NEXT_TASK.md。P2-07 的 ADR-0016、独立计划和工作日志已建立，不要重复设计或扩大范围。先只实现计划步骤 2 的最小只读 recovery verifier 及其目标测试；完成、复核并记录后再决定是否进入 Redis/Worker 或 rules/harness。自动化只用 Mock/Stub，所有 destructive 操作只针对隔离、精确目标；Phase 2 在 P2-07 完成前保持 in_progress。 ``` diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 7f09493..6473aa3 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -6,7 +6,7 @@ - Phase 0 — 项目治理和架构：`completed`（2026-08-24） - Phase 1 — MVP 垂直链路：`completed`（2026-08-25） -- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计、P2-01 单机资格与 P2-06 已完整交付；P2-07 为 `ready/next`，尚未实现） +- Phase 2 — 可靠性与任务执行：`in_progress`（可靠基础、治理/审计、P2-01 单机资格与 P2-06 已完整交付；P2-07 工作包已建立，状态为 `planned`，功能尚未实现） - Phase 3 — 标准 Benchmark 与代码评测：`in_progress`（仅可信本地 MMLU-Pro/GPQA-Diamond 客观题提前切片） - Phase 4–6：`planned` @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。Phase 2 仍为 `in_progress`，P2-07 为 `ready/next`、尚未实现。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -53,7 +53,7 @@ ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 -- P2-07：状态为 `ready/next`，尚未实现。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的完整运维演练尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 +- P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 @@ -117,4 +117,4 @@ ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 已完成仓库级收尾；下一任务为 `ready/next`、尚未实现的 P2-07 数据库/keyring backup/restore、Redis 重建、告警处置和剩余故障演练。Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 继续保持 `in_progress`。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d4bac63..ff0555b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,7 +30,7 @@ | --- | --- | --- | --- | --- | | Phase 0 | 项目治理和架构 | 可执行的需求、架构、协议、ADR 与持续文档流程 | `completed` | [PHASE-0-GOVERNANCE.md](phases/PHASE-0-GOVERNANCE.md) | | Phase 1 | MVP 垂直链路 | Mock 模型到 Run、逐题结果与排行榜的离线闭环 | `completed` | [PHASE-1-MVP.md](phases/PHASE-1-MVP.md) | -| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计、P2-01 与 P2-06 已交付；P2-07 为 `ready/next`，尚未实现 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | +| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计、P2-01 与 P2-06 已交付；P2-07 工作包已建立，状态为 `planned`，功能尚未实现 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | | Phase 3 | 标准 Benchmark 与代码评测 | 已有 MMLU-Pro/GPQA 可信本地切片；IFEval、沙箱与完整插件体系待完成 | `in_progress` | [PHASE-3-BENCHMARKS.md](phases/PHASE-3-BENCHMARKS.md) | | Phase 4 | Judge、Arena 与长上下文 | 可校准 Judge、Pairwise Judge、个人 Arena 和长上下文评测 | `planned` | [PHASE-4-JUDGE-ARENA.md](phases/PHASE-4-JUDGE-ARENA.md) | | Phase 5 | Agent、私有与 Live Benchmark | 工具调用轨迹、隔离私有集和持续更新的 Live Benchmark | `planned` | [PHASE-5-AGENT-LIVE.md](phases/PHASE-5-AGENT-LIVE.md) | @@ -207,7 +207,7 @@ | P2-04 生命周期可靠性 | 可靠基础已交付 | retry/取消/恢复/dead-letter/终态重算及三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；外部调用仍不保证 exactly-once | | P2-05 并发治理 | 切片已实现 | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序和完整性 fail-closed 已实现；真实 PG/capacity/acceptance/精确 SHA CI 候选门禁已通过 | | P2-06 可观测性 | `completed` | exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现；`9a20676…` 已 push、PR #3、实现 run `33164609388` 4/4，clean capacity/9/9 acceptance 全绿；evidence-doc commit `ec29596…` 已 push且精确 SHA run `33165775037` 4/4 | -| P2-07 验证与运维 | `ready/next` | P2-01/P2-06 已提供 capacity/PG/acceptance 基线，但 P2-07 本身尚未实现；下一步开展数据库+keyring backup/restore、Redis 重建、告警响应与完整失败矩阵 | +| P2-07 验证与运维 | `planned` | ADR-0016、独立计划与工作日志已建立；功能尚未实现，后续从最小只读 verifier 开始，再开展数据库+keyring restore、Redis 重建、告警响应与完整失败矩阵 | ### 验收标准 @@ -217,7 +217,7 @@ - `delivered`：历史 `0004` / 12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 - `completed`：P2-06 的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已在 SHA `9a20676dcf545040782f04c166205d0043345753` 实现；本地 lint/test/integration/rules 与修复后 76-file 技术/安全终审全绿，实现已 push 到 PR #3，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）9/9、Worker `2/2/2/0/0`、cleanup C/V/N empty；clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）记录 QPS `7.267474/12.962228/9.333604`、wall `8.255963/4.628834/6.428385s`、18/270/270/271/1230、0 question error/drift/duplicate/PEL/lag、expected=2/shortfall=0 与 cleanup C/V/N/image=0、image `1/1/0/0`。两者均为 clean-SHA Mock-only evidence，不是 SLO；此前 dirty evidence 保留为历史。Evidence-doc commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 成功，P2-06 仓库级收尾完成。 - `delivered`：`P2-local-control-plane-v2` 在 clean SHA `b6a35fe…` 完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 和 `qualified` 容量模型；只限定记录的 Mock-only 单机拓扑。 -- `ready/next`：P2-07 尚未实现；下一步开展 backup/restore、Redis 重建、告警响应与完整恢复矩阵。三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 +- `planned`：P2-07 工作包和恢复不变量已冻结，但功能尚未实现；后续开展 backup/restore、Redis 重建、告警响应与完整恢复矩阵。三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 - `delivered`：治理实现 SHA `665244e…` 已 push；本地 enhanced capacity、9/9 acceptance 与远程 run `33099260233` 4/4 均通过。 - `delivered`：P2-01 v2 实现 SHA `b6a35fe…` 已 push；本地 aggregate SHA-256 `a76d167b…d0d9` 为 23/23，实现 run `33146681285` 4/4 成功；证据文档 commit `875f13a…` 已 push，收尾 run `33150080341` 4/4 成功。 - `delivered`：P2-06 实现 SHA `9a20676…` 已 push 至 PR #3，实现 run `33164609388` 4/4 成功，绑定同一 SHA 的 clean capacity 与 9/9 acceptance 已通过；evidence closeout 文档 SHA `ec29596…` 的 run `33165775037` 也已 4/4 成功，仓库级收尾完成。 @@ -237,11 +237,11 @@ - 已交付候选切片：`0004`、历史 12 表 importer、四层治理/ledger/backpressure/fairness、typed audit/history/Provider/credential evidence、UI 状态、enhanced capacity/PG tests 与运维文档。 - 已完成 P2-06：`0005` / 13 表 importer、Worker generation/progress、Prometheus exporter 与八条规则、canonical audit retention CLI、公共 retained-row validator、全日志源治理，以及实现/evidence-doc 两个精确 SHA 的远程 CI。 - 治理候选门禁已完成；P2-01 的 v2 多轮资格、实现 commit/push、实现 SHA 4/4 CI 与证据文档精确 SHA 4/4 CI 也已完成，P2-01 仓库级收尾完成。 -- 阶段正式剩余：启动 `ready/next`、尚未实现的 P2-07 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 +- 阶段正式剩余：实施状态为 `planned` 的 P2-07 PostgreSQL+keyring backup/restore、Redis 重建、告警响应与完整故障矩阵。 ### 状态 -`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 的实现与 evidence-doc 精确 SHA CI 均已完成，状态为 `completed`；P2-07 为 `ready/next`，尚未实现。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 +`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 的实现与 evidence-doc 精确 SHA CI 均已完成，状态为 `completed`；P2-07 工作包已建立，状态为 `planned`，功能尚未实现。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 ## 6. Phase 3：标准 Benchmark 与代码评测 diff --git a/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md new file mode 100644 index 0000000..fd9317d --- /dev/null +++ b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md @@ -0,0 +1,216 @@ +# ADR-0016：PostgreSQL 与 keyring 配对恢复、Redis 重建及恢复演练 + +- **Status**: Accepted +- **Date**: 2026-08-28 +- **Deciders**: LLMBenchLab maintainers +- **Scope**: Phase 2 P2-07 备份/恢复验证、Redis rebuild、告警响应与故障矩阵 +- **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的恢复运维边界、[ADR-0007](ADR-0007-web-provider-credentials.md) 的数据库外 keyring 备份边界，以及 [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的告警与 archive 后续演练 +- **Preserves**: PostgreSQL 唯一任务事实来源、Redis 非权威通知、write-only Provider Key、`llmbenchlab-protocol-v1`、Mock-only 自动化和可信本地运维边界 + +## Context + +Phase 2 已交付 PostgreSQL 任务/治理事实、Redis Streams 通知、独立 Worker、租约/fencing、13 表 SQLite→PostgreSQL importer、数据库外 AES-GCM keyring、固定 exporter/八条规则和 audit archive。它们仍不能回答完整恢复问题： + +1. SQLite importer 是停写源到空 PostgreSQL 的一次性迁移，不是 PostgreSQL backup/restore；audit archive 只保存 typed audit 行，也不是整库备份。 +2. `model_credentials` 只保存 envelope。没有与备份点匹配的数据库外 keyring，即使 13 表完整恢复也不能恢复 stored Provider credential；只校验 key ID 或 active key 不能证明历史 envelope 可解密。 +3. Redis AOF、Stream、consumer group、PEL 和 lag 都不是任务事实。空 Redis 能由 Worker 的数据库扫描恢复，但必须证明丢失 group/PEL/lag 后仍能收敛，并避免提供 `FLUSHALL`、删 group 或“数据库重放到 Redis”这类危险生产命令。 +4. P2-06 的八条 Prometheus 规则已有静态合同和 Runbook，但尚未分别验证 `for` 时序、真实底层 symptom、处置动作与恢复证据。 +5. 既有 acceptance/capacity/crash-seam 不能冒充 PostgreSQL+keyring 恢复认证，也不能证明 PITR、RPO/RTO、HA、Alertmanager 投递或 Provider exactly-once。 + +本决定只为可信操作者、本地/隔离 PostgreSQL 16 与 Redis 7 拓扑建立可重复验证的恢复合同。它不把仓库变成生产备份平台。 + +## Decision + +### 1. 工具边界 + +- PostgreSQL 备份固定使用同 major 的 PostgreSQL 16 `pg_dump --format=custom`；恢复固定使用 `pg_restore --exit-on-error --single-transaction --no-owner --no-privileges`。 +- 仓库不实现生产级 dump/restore mutation CLI，不包办调度、异地复制、加密存储、PITR、WAL、保留策略或备份服务。`pg_restore --clean`、`--create`、覆盖既有数据库和自动 DROP/TRUNCATE 一律不属于受支持路径。 +- 新增只读运维入口 `llmbenchlab-recovery-verify`，只允许三个动作： + - `create-manifest`：读取已生成 dump、独立 keyring 和只读源数据库快照，安全 no-replace 写 canonical manifest；不修改数据库。 + - `artifacts`：完全离线核验 manifest、dump 与 keyring；不得导入数据库配置或构造 engine。 + - `restored-database`：只读核验已恢复数据库、manifest 与 keyring；不得创建、删除、迁移、清空或修正数据库。 +- `llmbenchlab-recovery-verify` 不接受 Provider Key、credential plaintext、DSN 或密码作为 argv；数据库连接只从明确环境变量/libpq service 取得。成功和失败只输出固定状态、匿名计数和 digest，不输出路径、URL、连接信息、对象 ID 或异常原文。 +- 新增独立 `make phase2-recovery` 隔离 Compose harness。它可以在随机且经过 label/identity guard 的项目内创建/删除测试数据库和 Redis volume；这些破坏性能力不进入生产 CLI 或 REST API。 +- 不新增 REST API、数据库 migration 或长期运行 backup 服务，也不新增生产依赖。 + +### 2. 维护窗口与 PostgreSQL 恢复不变量 + +- 创建备份前停止 admission、API、Worker、CLI mutation 和所有 application/audit writer；确认 Worker 已 graceful stop 或进入可解释 stale 状态。数据库健康探针可以继续只读连接，但任何未知 writer 都使备份集不合格。 +- `pg_dump` 与 source manifest snapshot 必须来自同一停写窗口。恢复后的 13 表摘要与 manifest 不一致即证明期间发生漂移或 artifact 不匹配，整组 fail closed。 +- 目标必须由操作者预先创建且没有任何用户 schema/table/Alembic row。先运行只读 empty-target 检查；目标非空时拒绝，绝不由工具清空。 +- 恢复后数据库必须已经精确位于 Alembic `20260828_0005`。禁止先升级旧 dump 再把它报告为原备份恢复成功；若未来支持新 head，必须在新 ADR/manifest schema 中显式声明兼容。 +- 13 张核心表固定为： + + ```text + governance_policies + models + model_credentials + benchmarks + questions + governance_scopes + evaluation_runs + evaluation_responses + governance_minute_buckets + question_executions + provider_call_reservations + audit_events + worker_processes + ``` + +- 每表必须逐项满足 `row_count`、主键集合 SHA-256 和 canonical 全行 SHA-256。恢复验证复用与 importer 相同的 typed canonical encoding，并重跑：policy hash/列、managed Run 冻结 snapshot/四 override、ledger→scope/minute 投影、active policy 唯一性、retained audit 完整事实、FK/约束和 Alembic head。 +- `worker_processes` stopped/stale facts、managed Run、Response、ledger 和 audit 必须原样保留；验证器不得通过清空或重建事实来制造一致。 +- P2-07 certification 源允许一个只使用 Mock 的 pending Run 来验证 Redis rebuild，但不得包含 running Run、active reservation 或 live Worker generation；这避免把未知 Provider 外部副作用混入恢复资格。例行备份也应优先排空或取消活动 Run。 +- 标准 PostgreSQL dump 本身可以保留生产故障点的 running/active 事实，但这类备份超出本次认证 profile；恢复时仍必须隔离目标、让 lease/reconciler 按既有规则收敛，并独立核对备份点之后的 Provider 副作用。 + +### 3. Canonical recovery manifest + +Manifest schema 固定为 `llmbenchlab-recovery-manifest-v1`，UTF-8、canonical JSON、未知/缺失字段、duplicate key、NaN/Infinity 和非规范时间均拒绝。最小字段为： + +```json +{ + "schema": "llmbenchlab-recovery-manifest-v1", + "backup_set_id": "00000000-0000-4000-8000-000000000000", + "created_at_utc": "2026-08-28T00:00:00.000000Z", + "source_alembic_head": "20260828_0005", + "source_git_commit": null, + "database_dump": { + "format": "postgresql-custom", + "postgres_server_major": 16, + "pg_dump_major": 16, + "size_bytes": 1, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "credential_keyring": { + "format": "llmbenchlab-credential-keyring-json-v1", + "size_bytes": 1, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "credential_binding": { + "stored_model_count": 0, + "envelope_count": 0, + "active_stored_run_count": 0 + }, + "tables": [] +} +``` + +- `backup_set_id` 为 canonical UUID v4；时间为六位微秒 UTC `Z`；Git SHA 为 40 位小写十六进制或 `null`。 +- `tables` 必须按上节顺序恰好出现 13 次，每项只含 `name/row_count/pk_set_digest/canonical_row_digest`。 +- Manifest 不包含路径、DSN、用户名/主机、key ID、Key、nonce、ciphertext、Provider URL、Run/Worker/Model ID、Prompt、Response 或日志。 +- Manifest 绑定完整 dump bytes 和完整 keyring bytes；不能只绑定 active key。数据库 dump/manifest 与 keyring 必须独立备份、分开存储并使用不同访问控制，配对时才临时同时可见。 +- Manifest 自身另有文件 SHA-256；operator 在验证和恢复时必须提交精确 digest。所有 SHA-256 只用于误改检测和精确配对，不是签名、来源认证、WORM 或恶意管理员防篡改证明。 +- 历史备份必须保留其完整配对 keyring，直到该备份按策略销毁。当前只支持添加新 key/切换 active；在没有批量重加密与独立轮换合同前，不得删除仍被当前或历史备份 envelope 引用的旧 key。 + +### 4. Artifact 文件安全 + +- Manifest、dump 与 keyring 输入必须是当前用户拥有的普通文件且 `st_nlink == 1`；拒绝 symlink、FIFO、device、socket、hard link、identity 变化以及 group/other 可读写的文件。权限不得宽于 `0600`。 +- 输入以 `O_NOFOLLOW` 单一 fd 打开，先后 `fstat` 校验 identity/size/mtime，采用固定 chunk 流式 hash；keyring 上限沿用 `64 KiB`，manifest 上限 `1 MiB`，certification dump 上限 `64 GiB`。任何读取期变化都 fail closed。 +- 输出父目录必须存在、当前用户拥有、非 symlink 且不可 group/other 写，建议 `0700`。Manifest 使用同目录 `O_EXCL|O_NOFOLLOW` 的 `0600` 临时文件、fsync、no-replace 原子安装和 parent fsync。 +- CLI parser、错误、日志和 evidence 不反射 argv path、异常、文件内容或工具 stdout/stderr。未知参数也只返回固定错误。 +- 自动化 keyring 只使用每次随机生成的 32-byte 假密钥；所有 Provider credential 环境变量从 child env 删除。 + +### 5. Keyring 配对与本地解密验证 + +验证顺序固定。恢复流程不得调用会在 keyring 缺失时自动生成新文件的 `make setup/dev/docker-up` 或 `bootstrap_credential_keyring.sh`；必须使用只验证既有 artifact 的恢复入口，且在配对验证成功前不得启动 API/Worker： + +1. `artifacts` 在任何数据库连接或 Adapter 构造前安全读取并严格解析 keyring，精确比较 manifest 的 size/SHA。即使数据库没有 stored credential，错误 keyring 也必须由 digest 拒绝。 +2. `restored-database` 对每个 `credential_source=stored` Model 构造现有 `EncryptedCredential`，用当前 Model ID/origin 作为 AAD 调用 `CredentialKeyring.decrypt()`；只累计成功数，plaintext 不比较、不 hash、不保存、不输出。 +3. 对每个 pending/running 且 frozen source 为 `stored` 的 Run，用 immutable Run snapshot 的 Model ID/origin 再验证一次；不得用当前可编辑 Model origin 代替。 +4. missing/unreadable、格式错误、digest mismatch、未知/退役 key、同 key ID 不同材料、错误 Model/origin、tampered nonce/ciphertext 均在 Adapter/HTTP 构造前稳定 fail closed，并证明 Provider I/O 次数为零。 + +稳定错误族为 `recovery_keyring_unavailable`、`recovery_keyring_invalid`、`recovery_keyring_digest_mismatch`、`recovery_credential_binding_failed`；不得包含 key ID 或底层 crypto 文本。 + +### 6. `pg_restore` 结果分类 + +- `pg_restore` exit 0 仍不表示恢复成功；必须再运行 `restored-database`，只有 13 表与 keyring 全 exact 才成功。 +- 工具非零或连接中断后，不盲目重跑、不执行 `--clean`： + - 能确认目标仍无用户对象：`precommit_failed`，可由操作者创建另一个空目标重试。 + - 目标与 manifest 全 exact：`committed_but_tool_or_postverify_failed`；保留目标并调查，不能再 restore。 + - 无法连接判断：`commit_outcome_unknown`；隔离目标，恢复连接后只读 reconcile。 + - 目标为 partial/mixed/conflict：`integrity_conflict`；保留现场，不自动删除或覆盖。 +- 只读 verifier 成功为 exit 0，安全拒绝为 exit 2。Compose harness 对上述 outcome 使用固定内部分类；exit 3/4 的既有语义分别保留给“已提交但后验证失败”和“提交结果未知”。 + +### 7. Redis replacement 与 Worker 恢复 + +- Redis rebuild 不恢复 AOF/Stream/consumer state，也不从数据库生成历史消息。平台提供空 replacement Redis 后，先启动一个 Worker：每轮先 `reap_expired()`/`due_run_ids()`，因此无需 Redis 即可恢复 pending/expired Run；随后 Worker 以 `XGROUP CREATE ... 0-0 MKSTREAM` 建立唯一 group。 +- 若 Stream 保留而 group 丢失，group 仍从 `0-0` 重建，让历史通知按 at-least-once 再投递；终态/重复通知必须由数据库 no-op 后 ACK。禁止用 `$` 跳过历史消息。 +- Redis/NOGROUP/连接失败必须令 Worker 重置本地 group-initialized flag 和 `XAUTOCLAIM` cursor；下次尝试重新初始化。数据库 scan 在整个 queue outage 期间继续。 +- 完全丢失 Redis 的演练至少保留：一个已完成基线、一个 orphan PEL、一个 lag pending，以及可选的部分 Response/过期 lease。重建后证明所有数据库事实收敛、新通知能完成 `XADD→XREADGROUP→XACK`、group 唯一、PEL/lag 为零、terminal duplicate 不改变 canonical snapshot、ledger projection 无漂移。 +- 生产代码不新增 `FLUSHDB/FLUSHALL`、删 Stream/group/volume 或 replay 命令。Harness 只能删除自身随机 project 下经 exact label/identity 校验的 Redis container/`redis-data` volume；PostgreSQL volume identity 删除前后必须相同。 +- 禁止 glob、未解析变量、`volume prune`、`image prune`、force、默认 project、共享 volume 和中途 `compose down -v`。最终 cleanup 才能对已验证随机 project 执行 scoped `down -v`。 + +### 8. Worker 扩缩与 expected minimum + +- `worker_expected_processes` 是 API/exporter 的部署声明，不由 Worker 或历史 generation 数推断。Compose 把该环境变量只注入 API，不再放入公共 backend environment。 +- 扩容 `1→2`：先增加 Worker 并确认两个 generation 已产生真实 DB scan，再以 expected=2 强制重建 API，最后验证 `expected/registered/live/stalled/shortfall=2/2/2/0/0`。 +- 缩容 `2→1`：先以 expected=1 重建 API，再 graceful scale Worker；等待被停 generation 写 `stopped_at` 或 lease 自然恢复，最后验证 `1/1/1/0/0`、Response 唯一和 ledger 收敛。 +- 当前资格只覆盖 1–2 Worker；第三个 Worker 或不同 pool/lease/profile 必须重新设计和测量。 + +### 9. 八规则时序与真实响应演练 + +P2-07 使用两层 AND 验收，不等待真实 15 分钟： + +1. 新增 Prometheus `promtool test rules` fixture，evaluation interval 固定 30 秒；八条规则都必须在 `for` 到期前不 firing、到期后 firing、输入恢复后 clear，并匹配固定 label/annotation。`run_dead_lettered`、integrity 和 overdrawn 是持久/rolling symptom，fixture 的 clear 只验证规则输入，不允许在真实数据库中删除事实制造“恢复”。 +2. 隔离 Compose 分别制造真实底层 symptom 并执行 Runbook 动作：API stop/restart、due backlog/drain、生产 repository failure budget→dead-letter、隔离库 projection drift→fail closed→从已验证备份恢复、Mock actual>reservation→overdraw、Redis outage/replacement、Worker stale/恢复、lease-owner kill/peer takeover。 + +真实演练证明 symptom、证据采集边界、处置和根因恢复；不声称仓库部署 Alertmanager sender、通知路由、silence API、ack 或生产值班系统。Dead-letter/integrity/overdrawn 恢复后只要求没有新增事件和新工作正常，不伪造旧事实消失。 + +### 10. 验证分层与 evidence + +- 单元测试覆盖 manifest canonical/path/权限/line/size/duplicate/future schema、keyring mismatch/crypto binding、只读 CLI/离线 import、result classifier、Redis NOGROUP reset、project/volume guard、evidence allowlist 和规则 fixture 合同。 +- 真实 PostgreSQL/Redis integration 覆盖 custom dump→空目标 restore→13 表 exact、credential local decrypt、nonempty target refusal、commit-unknown classifier、orphan PEL/XAUTOCLAIM、group destroy/recreate from `0-0`、ACK unknown/duplicate no-op 及故障矩阵的 repository/ledger 状态转换。 +- 独立 Compose recovery harness 覆盖完整 DB+keyring 配对、空 Redis volume replacement、Worker 1↔2、八项底层 symptom/响应与 scoped cleanup；保留现有 P2-06 9/9 acceptance 和 P2-01 SLO 合同不变。 +- `phase2-recovery` 使用严格 allowlist evidence。内部/CI summary 只保存 schema、suite UTC 起止、commit/dirty、脚本/Compose/rules hash、PG/Redis major、scenario 名称/状态/时长、固定 revision、`table_count=13`、整体 count/PK/content/governance/audit/keyring equality、稳定 outcome、匿名聚合计数、Worker/Redis/ledger/cleanup count 和布尔门禁；不保存逐表 count/hash、命令、stdout/stderr、日志、traceback、路径、DSN/URL、port、对象/container/image/volume/database ID、环境名/值、key ID、keyring hash/material、envelope、题目、Prompt 或 Response。 +- 公开文档只引用外层 evidence 相对路径、整文件 SHA-256、commit/dirty、匿名汇总和边界；raw evidence 不提交、不上传。CI artifact 若保留，只允许相同 allowlist schema。 +- P2-07 实现进入现有 `Real Compose reliability acceptance` 必需 job，但 job 名保持不变；该 job 先运行冻结的 P2-06 acceptance，再运行 P2-07 recovery harness。P2-07 只上传通过 schema validator 的 allowlist summary，dump/manifest/keyring/raw evidence 永不上传。GitHub-hosted runner 的结果证明功能门禁，不是恢复 SLA。 + +## Alternatives considered + +### 自制 PostgreSQL backup/restore mutation CLI + +拒绝。它会复制 PostgreSQL 工具、扩大权限/版本/commit outcome 面，并诱导操作者把仓库脚本当生产 DR 平台。 + +### 把 keyring 嵌入 dump 或与 dump 存在同一 bundle + +拒绝。数据库与 keyring 同时泄漏即可恢复 Provider Key；独立存储和访问控制是现有凭据安全边界的一部分。 + +### 只比较 key ID 或只尝试解密现有 envelope + +拒绝。零 stored credential 时会让错误 keyring 空集通过；只比较 ID 也不能发现同 ID 不同 key material。必须先比较完整 artifact digest，再执行 AES-GCM/AAD 验证。 + +### 备份/恢复 Redis 或把数据库状态 replay 为 Stream + +拒绝。Redis 不是任务事实来源；恢复历史 queue state 只会增加重复和错误权威性。空 Redis + DB reconciliation 是更小且可证明的恢复路径。 + +### 真实等待八条告警的 `for` 时长 + +拒绝。15 分钟 wall-clock 等待会让 CI 变慢且不增加表达式可信度；使用 promtool synthetic time 验证规则时序、Compose 验证真实 symptom/Runbook。 + +## Consequences + +### Positive + +- 备份与恢复职责保持在成熟 PostgreSQL 工具，仓库验证器只读、可复用现有 13 表/治理/audit 合同。 +- 数据库与 keyring 既能精确配对，又保持分离存储和不同访问控制。 +- Redis 全丢失、group 丢失和重复通知的恢复路径与数据库权威设计一致，不引入危险生产删除命令。 +- 八条规则同时获得确定性时序测试和真实底层响应演练。 +- Evidence 默认比既有 acceptance 更小，降低 dump、凭据和基础设施信息泄漏面。 + +### Negative + +- 操作者仍需自行部署、调度、加密和异地保存 PostgreSQL dump/keyring；仓库不提供一键生产恢复。 +- 配对验证需要在受控环境临时同时访问 dump、manifest 和 keyring。 +- 完整 Compose recovery 会增加本地/CI 时间，并需要严格清理随机数据库、volume 和临时文件。 +- 备份点之后已发生的 Provider 请求无法由数据库恢复；本地恢复仍不是 exactly-once。 + +## Rollback + +- 只读 verifier 和 recovery harness 可从调用面停用，不影响数据库 schema/API/评分协议；manifest/dump/keyring 保持敏感资产，不能因代码回滚自动删除。 +- Compose expected 环境重定位若回滚，必须同步恢复部署文档并承认运行中 API 的 expected 不会随 Worker scale 自动变化。 +- Redis queue 修复只能回退代码，不能删除数据库事实或用旧 AOF 覆盖 PostgreSQL。 +- 已恢复目标出现不确定结果时保持隔离；禁止通过 DROP/clean 伪装回滚。删除测试目标只限 harness 的 exact project/database guard 与最终 cleanup。 + +## Follow-up + +- 若未来需要生产 PITR、RPO/RTO、签名 manifest、KMS/HSM、异地对象锁、自动轮换/重加密或多主机 HA，必须另立 ADR 与部署/安全模型。 +- P2-07 完成只允许评估 Phase 2 的约定范围；不得外推真实 Provider、第三个 Worker、公共多租户、Alertmanager 或灾难恢复 SLA。 diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index b580ca1..5b38a2a 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -66,7 +66,7 @@ | P2-04 生命周期可靠性 | `foundation_delivered` | retry/backoff、取消、恢复、dead-letter、Response 幂等和三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；Provider 外部副作用仍为 at-least-once | | P2-05 并发治理 | `slice_delivered` | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序、counter 重算 fail-closed 与 ADR-0011 已实现；精确 SHA 的真实 PG/capacity/acceptance/CI 候选门禁已通过 | | P2-06 可观测性 | `completed` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已进入 clean commit `9a20676…`；clean capacity/9/9 acceptance、实现 CI 与 evidence-doc commit `ec29596…` 的 CI 4/4 均通过 | -| P2-07 验证与运维 | `ready/next` | 既有 enhanced capacity/PG tests、正式 v2 单机资格与运维文档是共享前置证据；本切片尚未实现，下一步须先新建独立计划/日志/必要 ADR，再实施 PostgreSQL+keyring backup/restore、Redis 重建、完整失败矩阵与告警响应 | +| P2-07 验证与运维 | `planned` | [ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、[独立计划](../plans/2026-08-28-phase-2-recovery-operations.md) 与 [工作日志](../worklogs/2026-08-28-phase-2-recovery-operations.md) 已建立；功能尚未实现，后续从最小只读 verifier 开始 | `slice_delivered` 表示该垂直切片及其候选门禁已交付，不表示整个阶段完成。Phase 2 必须保持 `in_progress`。 @@ -135,8 +135,8 @@ ## 交付物与下一任务 -已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 与 evidence-doc SHA `ec29596…` 已分别通过精确 4/4 CI，仓库级闭环完成；下一步按 [NEXT_TASK.md](../NEXT_TASK.md) 新建 P2-07 独立计划/日志/必要 ADR，再实施 backup/restore 和剩余运维演练。 +已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 与 evidence-doc SHA `ec29596…` 已分别通过精确 4/4 CI，仓库级闭环完成；P2-07 的 ADR/计划/日志已建立，后续按 [NEXT_TASK.md](../NEXT_TASK.md) 从最小只读 verifier 开始实施。 ## 状态 -`in_progress`。P2-01 与 P2-06 已完成，P2-05 已交付；P2-07 为 `ready/next` 但尚未实现，数据库+keyring backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +`in_progress`。P2-01 与 P2-06 已完成，P2-05 已交付；P2-07 状态为 `planned`，功能尚未实现，数据库+keyring backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 diff --git a/docs/plans/2026-08-28-phase-2-recovery-operations.md b/docs/plans/2026-08-28-phase-2-recovery-operations.md new file mode 100644 index 0000000..6157dc1 --- /dev/null +++ b/docs/plans/2026-08-28-phase-2-recovery-operations.md @@ -0,0 +1,140 @@ +# Phase 2 恢复与运维闭环执行计划 + +- Owner: Codex +- Status: in_progress +- Created: 2026-08-28 +- Updated: 2026-08-28 +- Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- Worklog: [2026-08-28 工作日志](../worklogs/2026-08-28-phase-2-recovery-operations.md) +- ADRs: [ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0007](../decisions/ADR-0007-web-provider-credentials.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md)、[ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) + +## Context + +P2-01 和 P2-06 已分别完成单机控制面资格与可观测性/retention 仓库级门禁。当前 PostgreSQL 是任务唯一事实来源，Redis 只是通知；13 表 importer、数据库外 keyring、audit archive、Worker DB-time progress、固定 exporter/八规则和 9/9 acceptance 均已实现，但仓库尚未认证 PostgreSQL+keyring 配对恢复、空 Redis rebuild、规则时序/响应和剩余真实故障矩阵。P2-07 是 Phase 2 的最后约定切片，必须使用新的计划/日志/ADR，不复用 P2-06 evidence 冒充恢复认证。 + +## Objective + +在不调用真实 Provider、不改变 API/评分协议、不提供危险生产 mutation 命令的前提下，交付 P2-07：PostgreSQL 16 custom dump 与独立 keyring 的 canonical 配对验证、空目标 13 表精确恢复、Redis replacement 的数据库恢复路径、1↔2 Worker/八规则响应及真实 PostgreSQL/Redis 故障矩阵，并以独立 commit、clean-SHA evidence、push 和精确 SHA CI 完成仓库级闭环。 + +## Scope + +- 抽取 importer 的 13 表 canonical snapshot 与完整性检查为公共只读恢复基础。 +- 严格 recovery manifest、artifact/keyring 文件边界和只读 `llmbenchlab-recovery-verify`。 +- 标准 PostgreSQL 16 `pg_dump`/`pg_restore` 运维合同与空目标/结果分类，不自制 restore mutation。 +- Redis group/NOGROUP/replacement、数据库 scan、orphan PEL/lag/duplicate 的真实恢复验证。 +- Compose expected minimum 与 1↔2 Worker 扩缩顺序修正。 +- 八规则 promtool 时序 fixture 与真实底层 symptom/Runbook 演练。 +- dead-letter、commit outcome unknown、governance integrity/overdraw、cancel/retry/lease/budget 等剩余真实 PostgreSQL/Redis 故障矩阵。 +- 独立 `phase2-recovery` harness、Make/CI 接线、严格 evidence allowlist 与相关运维/安全/测试/状态文档。 + +## Non-goals + +- 生产 backup scheduler、WAL/PITR、RPO/RTO、HA、多区域、Kubernetes、对象存储、KMS/HSM 或签名/WORM。 +- PostgreSQL/Redis destructive REST API、通用 DROP/TRUNCATE、`pg_restore --clean/--create`、`FLUSHALL`、删 Stream/group 或 DB→Redis replay。 +- Alertmanager sender、通知路由、silence/ack 系统、值班平台或生产告警 SLA。 +- 第三个 Worker、不同硬件/profile、真实 Provider、费用/exactly-once 或公共多租户。 +- 批量 credential 重加密、旧 key 自动删除、Benchmark/协议/评分/前端产品功能变化。 + +## Assumptions + +- PostgreSQL 16 官方工具是 dump/restore 的事实实现；仓库只验证工件、数据库和 keyring，不重复实现数据库恢复。 +- 恢复源在维护窗口停写；restore 目标为随机、隔离、预创建的空数据库。 +- keyring 与数据库 dump 分开存储、使用不同访问控制；manifest 是敏感内部配对元数据，SHA 不是认证。 +- 自动化只使用 Demo/Mock、假 stored credential 和隔离 PostgreSQL/Redis；不会构造真实 Provider Adapter/HTTP 请求。 +- GitHub Actions 现有四个 required job 名保持不变；P2-07 harness 加入 full-stack job。 + +## Requirements + +- [NEXT_TASK P2-07](../NEXT_TASK.md) 的 backup/keyring、Redis rebuild、alert response 与 fault matrix 范围。 +- AGENTS §2 的 ADR/计划/日志先行、§3.2 的安全/部署/文档联动、§4 验证和 §5 exact-SHA 远程闭环。 +- ADR-0005 的 PostgreSQL truth/Redis notification、ADR-0007 的数据库外 keyring、ADR-0009/0015 的 ledger/audit/retention 和 ADR-0016 的恢复合同。 +- 现有 API、`llmbenchlab-protocol-v1`、P2-06 9/9 acceptance 与 P2-01 SLO 合同保持兼容。 + +## Implementation steps + +1. [completed] 完成只读勘察并冻结 ADR-0016。 + - Files/modules: keyring/credentials、importer/snapshot、queue/Worker、Compose/CI、alert rules/Runbook、acceptance/capacity。 + - Result: 标准 PG16 工具 + 只读 verifier、分离 keyring、canonical manifest、空目标、Redis replacement、八规则双层演练、destructive/evidence guard 已冻结。 + - Validation: 核对 P2-06 clean 状态、现有 13 表/queue/Worker/alert 实现及文件/测试边界；无工作区重叠。 +2. [pending] 实现共享恢复完整性、manifest 与只读 verifier。 + - Files/modules: `backend/app/db/recovery_integrity.py`、`backend/app/recovery/`、`backend/app/cli/recovery_verify.py`、`pyproject.toml`、`import_sqlite.py` 和目标测试。 + - Validation: canonical round-trip、13 表固定顺序、strict JSON、offline import、nofollow/owner/mode/size/race、dump/keyring swap、wrong/missing/tampered keyring、Model/active Run AAD、零 Provider I/O、结果/错误输出固定。 +3. [pending] 实现 Redis rebuild、Worker expected 扩缩与真实 integration。 + - Files/modules: `compose.yaml`、Worker/queue、Redis integration/Worker tests。 + - Validation: NOGROUP reset、group `0-0` rebuild、orphan PEL/XAUTOCLAIM、ACK unknown/duplicate no-op、DB scan、1↔2 expected/live/stopped/ledger 收敛。 +4. [pending] 实现八规则时序 fixture与 P2-07 recovery harness。 + - Files/modules: `deploy/observability/`、`scripts/phase2_recovery.py`、Makefile、CI、script tests。 + - Validation: promtool 八规则 before/fire/clear；PG custom dump→空目标 restore→13/13 exact；matching/wrong/missing keyring；Redis volume replacement；八 symptom/Runbook；dead-letter/integrity/overdraw/lease/cancel/retry/budget/commit-unknown；strict allowlist evidence/scoped cleanup。 +5. [pending] 完成代码/安全/文档联动与完整本地门禁。 + - Files/modules: README、Architecture、Security、Deployment、Operations、Testing、CHANGELOG、PROJECT_STATUS、Roadmap、Phase 2、NEXT_TASK、本计划/日志。 + - Validation: target/full pytest、Ruff、frontend、smoke、双方言 migration、real PG/Redis、promtool、Compose config、P2-06 regression、P2-07 recovery、链接/secret/diff/staged B/H/M review。 +6. [pending] 形成 clean implementation/evidence/status commit 并完成远程门禁。 + - Validation: 独立 implementation commit/push；在该 clean SHA 从零运行正式 P2-07 recovery evidence；证据文档 commit/push；每个精确 SHA 的四个必需 GitHub Actions job 全 success；Phase 2 在最后门禁前保持 `in_progress`。 + +## Risks + +| 风险 | 可能性/影响 | 预防措施 | 触发后的处理 | +|---|---|---|---| +| restore 覆盖用户数据库 | 低/极高 | verifier 只读、空目标、随机隔离名、禁 clean/create/drop/truncate | 停止流程，保留目标，人工确认；不自动清空 | +| dump 与 manifest 不是同一停写快照 | 中/高 | 维护窗口停 writer、恢复后 13 表 exact comparison | 整组 fail closed，重新建立新备份集 | +| DB/keyring 一起泄漏 | 中/极高 | 分开存储/ACL、短时配对、evidence 不含内容或指纹 | 停止传播，轮换 Provider credential 和 keyring；保留审计 | +| wrong keyring 在零 envelope 时空集通过 | 中/高 | 先比较完整 keyring bytes digest，再做 AES-GCM/AAD | 拒绝恢复，不构造 Adapter/HTTP | +| Redis 删除扩大到 PG/共享 volume | 低/极高 | exact project/container/volume label+identity，禁 glob/prune/down-v 中途 | 拒绝删除；只报告固定 cleanup failure | +| 新 Redis 后 Worker 永久认为 group 已初始化 | 中/高 | queue failure 重置 initialized/cursor，NOGROUP/real rebuild 回归 | DB scan 继续，修复初始化后再恢复通知 | +| 告警演练靠缩短/删除事实“碰绿” | 中/高 | promtool synthetic time + 真实 symptom 双层 AND；持久事实不清除 | 保留失败 evidence，修合同后从零重跑 | +| Compose/CI 时间或噪声过大 | 中/中 | 新独立 harness、固定有界场景、保持 required job 名 | 记录真实失败，优化确定性，不降低断言 | +| 备份点后 Provider 外部副作用丢失 | 中/高 | 明确非 exactly-once，保守 settlement 与账单/request ID 对账 | 停止新流量，人工外部对账，不能由 Redis 补造 | + +## Validation + +| 验收项 | 命令或检查 | 预期结果 | 实际结果 | +|---|---|---|---| +| Manifest/verifier | 目标 pytest + CLI subprocess | strict/canonical/offline/secret-safe 全通过 | 待执行 | +| Recovery integrity | SQLite 单测 + PostgreSQL integration | 13 表与治理/audit/keyring 全 exact | 待执行 | +| Redis/Worker | unit + real Redis/PostgreSQL integration | NOGROUP/group/PEL/duplicate/scale 收敛 | 待执行 | +| Alert rules | parser tests + Prometheus `promtool test rules` | 八规则 before/fire/clear 全通过 | 待执行 | +| P2-07 Compose | `make phase2-recovery` | backup/restore/rebuild/drill/fault/cleanup 全通过 | 待执行 | +| P2-06 regressions | `make phase2-acceptance`、目标 capacity | 冻结 9/9 与既有 capacity 合同不回归 | 待执行 | +| Backend | target + full `pytest` | 全部通过，integration 环境项明确 | 待执行 | +| Lint/build | `make lint`、frontend test/build | 全部通过 | 待执行 | +| Smoke/deployment | `make smoke`、`docker compose config --quiet` | Mock-only、配置合法 | 待执行 | +| Security/docs | staged secret/destructive/evidence scan、Markdown links/diff | 0 未解决 Blocker/High/Medium | 待执行 | +| Remote exact SHA | 普通 push + GitHub Actions | 四个 required job 对每个精确 SHA 全绿 | 待执行 | + +## Rollback + +先停止 recovery harness 和 verifier 调用。只读 verifier/manifest 代码可回滚而不改变 schema/API；已经创建的 dump、manifest、keyring backup 和恢复目标是敏感运维资产，不能由代码回滚自动删除。Redis/Worker 修复可回退代码但不得覆盖 PostgreSQL 或删除 durable facts。任何 restore commit outcome unknown、partial target 或 cleanup failure 都保持隔离，禁止 `--clean`/DROP/force；由只读 verifier 与操作者决定后续。Compose expected 环境重定位若回滚，必须同时还原部署文档并披露 running API 不会自动更新 expected。 + +## Documentation updates + +- [ ] README / 用户恢复入口和非 DR 边界 +- [ ] Architecture / PostgreSQL truth、配对 manifest、Redis rebuild +- [ ] Security / dump+keyring 分离、文件/evidence/CLI 输出边界 +- [ ] Deployment / PG16 标准工具、空目标、Worker scale 顺序 +- [ ] Operations / 完整 backup/restore/Redis/八规则 Runbook 演练步骤 +- [ ] Testing / test layers、Mock-only、real PG/Redis 与 evidence +- [ ] CHANGELOG、PROJECT_STATUS、Roadmap、Phase 2、NEXT_TASK、工作日志 +- [ ] GitHub workflow / 保持 required job 名、只上传 recovery allowlist summary，并修正历史 acceptance 场景数口径 +- [ ] API / 不适用：不新增或改变 REST 路径/schema/status；在计划/日志记录此判断 +- [ ] Benchmark/Data format / 不适用：协议、数据 schema、评分不变 + +## Completion evidence + +- Changed files: 待实施后记录。 +- Commands run: 待实际执行后记录。 +- Acceptance evidence: 待 clean implementation SHA 运行后记录；dirty/preflight 结果不得冒充正式证据。 +- Remote evidence: 待 commit/push/exact-SHA CI 后记录。 +- Not run: 当前尚未进入实施验证。 +- Known issues: Phase 2 仍为 `in_progress`；P2-07 未完成，不能宣称生产 DR、PITR/RPO/RTO、HA、Alertmanager 或 Provider exactly-once。 + +## Decision and discovery log + +| 日期 | 类型 | 记录 | 影响/后续 | +|---|---|---|---| +| 2026-08-28 | discovery | P2-06 implementation/evidence/status 三个 SHA 均已 push 且精确 SHA CI 4/4；工作树 clean。 | 可启动独立 P2-07，不复用 P2-06 dirty/clean artifact 冒充恢复证据。 | +| 2026-08-28 | discovery | Backend image 不包含 host `pg_dump/pg_restore`；Compose PostgreSQL 16 image包含标准工具。 | Harness 使用隔离 PG container；生产文档要求部署提供同 major 标准工具。 | +| 2026-08-28 | discovery | importer 已有 13 表 canonical summary 与 governance/audit preflight，但语义仍是 SQLite→PG mutation。 | 抽取公共只读 integrity 模块，避免 recovery verifier 依赖 importer 私有函数。 | +| 2026-08-28 | discovery | Worker queue failure 已重置 initialized/cursor；真实 NOGROUP/replacement 路径仍缺回归。 | 保持实现语义并补 unit/real Redis/Compose 证据。 | +| 2026-08-28 | discovery | running API 的 expected minimum 不会随 shell 中的 Worker scale 自动更新。 | Compose 只向 API 注入 expected，扩缩时按 ADR-0016 顺序重建 API。 | +| 2026-08-28 | decision | ADR-0016 拒绝生产 restore/Redis delete API，冻结标准 PG16 工具、只读 verifier、分离 keyring、空目标和 allowlist evidence。 | 实施分为 verifier、Redis/Worker、rules/harness 三条线。 | +| 2026-08-28 | scope | 用户要求避免过度工程化，本轮只落实最近的 P2-07 启动任务并停止。 | 本轮只提交 ADR、计划、工作日志和必要状态同步；步骤 2～6 保持 `pending`，不产生代码、脚本、Compose 或 CI 行为变更。 | diff --git a/docs/worklogs/2026-08-28-phase-2-recovery-operations.md b/docs/worklogs/2026-08-28-phase-2-recovery-operations.md new file mode 100644 index 0000000..b9cec9a --- /dev/null +++ b/docs/worklogs/2026-08-28-phase-2-recovery-operations.md @@ -0,0 +1,110 @@ +# 2026-08-28 — Phase 2 恢复与运维闭环工作日志 + +## 元信息 + +- 日期：2026-08-28 +- 执行者：Codex +- 分支：`codex/complete-evaluation-workflow` +- 初始 HEAD：`fc7bc9bf406c15920aede0be08bb128c25da277f` +- 关联阶段：[Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- 关联计划：[Phase 2 恢复与运维闭环执行计划](../plans/2026-08-28-phase-2-recovery-operations.md) +- 决策基础：[ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0007](../decisions/ADR-0007-web-provider-credentials.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) +- 本轮决策：[ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) +- 当前状态：P2-07 启动工作包 `completed`；P2-07 功能实现仍为 `not_started` + +## 目标与背景 + +P2-06 已在 implementation SHA `9a20676dcf545040782f04c166205d0043345753`、evidence-doc SHA `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 和最终状态 SHA `fc7bc9bf406c15920aede0be08bb128c25da277f` 分别完成普通 push 与精确 SHA 4/4 CI。Phase 2 仍缺 P2-07：数据库+keyring 配对恢复、Redis replacement、八规则响应和剩余故障矩阵。 + +本轮按用户要求只落实最近的启动任务：在不调用真实 Provider、不改变评分/API 协议、不引入危险生产恢复命令的前提下，完成只读勘察，冻结恢复合同，并建立独立计划和工作日志。只读 verifier、隔离恢复、Redis/Worker/alerts/fault matrix 均留给后续独立实施，不在本轮展开。 + +## 后续计划范围 + +- ADR-0016、独立计划/日志。 +- 13 表 canonical recovery integrity 与 strict manifest/artifact/keyring verifier。 +- PostgreSQL 16 custom dump→空目标 restore→read-only exact verify。 +- matching/missing/wrong keyring 的 AES-GCM/AAD 本地验证，零 Provider I/O。 +- Redis volume/group/PEL/lag 丢失后的数据库扫描恢复和新通知验证。 +- Worker expected minimum 与 1↔2 扩缩顺序。 +- 八规则 promtool 时序、真实底层 symptom/Runbook 和故障矩阵。 +- Make/CI/evidence/security/docs/commit/push/exact-SHA gate。 + +## 非目标 + +- 生产 backup scheduler、PITR/WAL、RPO/RTO、HA、Kubernetes、对象锁/KMS/WORM。 +- PostgreSQL/Redis destructive API、自制 restore CLI、`--clean/--create`、FLUSH/删 group/DB replay。 +- Alertmanager sender/通知/ack/silence 系统。 +- 真实 Provider、第三个 Worker、公共多租户、批量 credential 重加密。 +- Benchmark、protocol、评分、前端产品或 REST API 行为变化。 + +## 验收标准 + +- 正确 dump+keyring 在新空 PG16 目标恢复后 head=`20260828_0005`、13 表 count/PK/content digest 与治理/audit 完整性全 exact。 +- Manifest/dump/keyring swap、missing/wrong/权限/类型/size/race/tamper 全部 fail closed；stored fake credential 全部本地解密且 Provider I/O=0。 +- Redis container/volume 完全替换后，DB-only scan 恢复 pending/expired、group 从 `0-0` 重建、PEL/lag=0、重复通知不改变 durable snapshot。 +- Worker 1↔2 扩缩的 expected/live/stalled/shortfall、graceful stop、lease/fencing/ledger/Response 唯一性收敛。 +- 八条规则都有 synthetic before/fire/clear 与真实 symptom/Runbook 处置证据；持久事实不被删除伪造恢复。 +- remaining dead-letter、commit unknown、integrity/overdraw、cancel/retry/lease/budget matrix 在 real PG/Redis/Mock-only 环境通过。 +- 全量/lint/smoke/build/migration/Compose/secret/diff/staged review、独立 commit/push 和每个 exact SHA CI 全绿；在此之前 P2-07/Phase 2 保持 `in_progress`。 + +## 假设 + +- 标准 PostgreSQL 16 工具由部署环境/PG container 提供；backend 不捆绑数据库恢复工具。 +- 源数据库在明确维护窗口停止所有 writer，目标数据库新建且为空。 +- keyring 与 dump 分离存储，只在受控验证时同时可读。 +- 自动化仅使用随机假 keyring、stored credential canary、Mock/Stub/MockTransport。 + +## 风险 + +- 误删默认/共享数据库或 volume：全部 destructive 操作只允许随机隔离 project/database、exact label/identity，禁止 glob/prune/force。 +- dump/keyring/manifest 泄漏：0600/0700、nofollow/no-replace、分离 ACL、严格 evidence allowlist，不记录路径/命令/内容/ID。 +- restore commit outcome unknown：不盲重试或 clean；保留隔离目标并只读分类。 +- 错误 keyring 空集通过：artifact digest 必须先精确匹配，再做 envelope decrypt。 +- Redis 被误当任务备份：只允许空 replacement + DB reconciliation，不恢复 AOF/PEL 权威状态。 +- 告警/CI 为快速过绿而降低语义：规则时序与真实 symptom 双层 AND，失败 evidence 保留并从零重跑。 + +## 实施步骤 + +1. 阅读强制文档、检查 clean 工作树并勘察 keyring/importer/queue/Worker/alerts/Compose/CI。 +2. 新增 ADR-0016、执行计划和本日志，冻结工具、安全、结果和 evidence 边界。 +3. 抽取 recovery integrity，实现 manifest/只读 verifier 与单元/PG integration。 +4. 补 Redis NOGROUP/rebuild、Worker expected/scale 与 real Redis/PG 测试。 +5. 实现 promtool fixture、独立 Compose recovery harness、八响应/fault matrix 和 scoped cleanup。 +6. 同步文档，运行完整门禁，完成 staged 技术/安全审查。 +7. 独立 commit/push，在 clean SHA 重跑正式 evidence，完成 evidence/status commit 与各自 exact-SHA CI。 + +## 初始勘察事实 + +- 初始工作树 clean，HEAD 与 origin 均为 `fc7bc9bf406c15920aede0be08bb128c25da277f`。 +- 当前 Alembic head 为 `20260828_0005`，importer 固定 13 表并已有 typed canonical digest、policy/Run/ledger/audit preflight。 +- 本机没有 host `pg_dump/pg_restore`，Docker/Compose 可用；隔离 harness 应使用 PostgreSQL 16 container 内标准工具。 +- `CredentialKeyring` 已提供严格 JSON、最多 32 个 32-byte key、AES-GCM 与 Model/origin AAD；当前运行时读取不等同备份 artifact 的 owner/mode/race 检查。 +- 现有 setup/dev/docker-up 会在 keyring 缺失时自动生成新文件；恢复路径必须绕过 bootstrap，只验证既有配对 artifact，并在验证成功前禁止启动 API/Worker。 +- Worker 每轮先 reaper/due DB scan，再读 Redis；queue failure 已把 initialized flag 与 autoclaim cursor 重置，但 Redis replacement/group-loss 缺少真实回归。 +- P2-06 八条 rule/Runbook 已交付；真实 15 分钟等待不适合作为 CI，需 promtool synthetic time + Compose symptom 双层验收。 +- `worker_expected_processes` 由 API Settings 输出；只 scale Worker 不会更新运行中 API 的 expected。 +- P2-07 不需要 REST API；`docs/API.md` 预计不改，最终日志仍须记录此适用性判断。 + +## 实际修改 + +- 新增 ADR-0016、P2-07 独立计划和本工作日志。 +- 最小同步 README、Changelog、项目状态、Roadmap、Phase 2 和 NEXT_TASK，使仓库明确 P2-07 已有工作包但功能尚未开始。 +- 未修改生产代码、测试、Compose、CI、API、数据库 schema 或依赖。 + +## 已运行命令与结果 + +- `git status --short --branch`：初始工作树 clean，分支与 origin 同步。 +- 强制文档/ADR/API/Operations/Deployment/Testing 与 keyring/importer/queue/Worker/Compose/CI 只读勘察：完成。 +- `git diff --check`、三个新增文件的 no-index whitespace 检查与 9 份变更文档的 97 个相对链接检查：通过，0 缺失。 +- staged 范围/模式/秘密/禁止 artifact 路径检查：仅 9 份预期 Markdown，新增文件均为普通 `100644` 文本，未发现真实凭据、绝对本机路径或 evidence/artifact。 +- 代码测试/lint/Compose：未运行；本轮只有文档启动包，不得写成通过。 + +## 已知问题与未完成项 + +- P2-07 所有功能实现/验证仍未开始；当前仅合同、执行记录和状态入口落盘。 +- Phase 2 必须保持 `in_progress`；不得宣称 DR SLA、PITR/RPO/RTO、HA、Alertmanager 或 Provider exactly-once。 +- 默认用户 SQLite 不在本任务范围，不迁移、不删除、不作为恢复目标。 + +## 下一步 + +本轮在启动工作包提交、push 和精确 SHA CI 后停止。后续恢复本任务时，从计划步骤 2 的最小只读 verifier 切片开始；不得把本轮文档提交解释为 P2-07 功能已实现。 From 8fb51b690ae6335b8ef93b3cbe54e039781fb173 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 00:35:31 +0800 Subject: [PATCH 05/12] fix: repair legacy governance index migrations --- CHANGELOG.md | 1 + README.md | 2 +- ...20260829_0006_repair_governance_indexes.py | 125 ++++++ backend/app/db/prepare_migrations.py | 117 +++++- backend/app/governance/audit_archive.py | 7 +- backend/tests/test_audit_archive.py | 11 + backend/tests/test_migrations.py | 370 +++++++++++++++++- .../tests/test_phase2_acceptance_script.py | 1 + backend/tests/test_phase2_capacity_script.py | 1 + docs/ARCHITECTURE.md | 2 +- docs/DEPLOYMENT.md | 6 +- docs/NEXT_TASK.md | 10 +- docs/OPERATIONS.md | 6 +- docs/PROJECT_STATUS.md | 20 +- docs/SECURITY.md | 2 +- docs/TESTING.md | 9 +- ...ability-worker-progress-audit-retention.md | 7 +- ...esql-keyring-recovery-and-redis-rebuild.md | 7 +- ...hema-equivalent-governance-index-repair.md | 49 +++ docs/phases/PHASE-2-RELIABILITY.md | 5 +- .../2026-08-28-phase-2-recovery-operations.md | 4 +- .../2026-08-29-database-startup-recovery.md | 108 +++++ .../2026-08-28-phase-2-recovery-operations.md | 2 +- .../2026-08-29-database-startup-recovery.md | 90 +++++ scripts/phase2_acceptance.py | 9 +- 25 files changed, 916 insertions(+), 55 deletions(-) create mode 100644 backend/alembic/versions/20260829_0006_repair_governance_indexes.py create mode 100644 docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md create mode 100644 docs/plans/2026-08-29-database-startup-recovery.md create mode 100644 docs/worklogs/2026-08-29-database-startup-recovery.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a747c2b..593431f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Fixed +- Added ADR-0017 and the forward-only, schema-equivalent Alembic repair revision `20260829_0006` for databases that executed an early `0004` variant before three governance indexes were present. Migration preflight accepts canonical schemas or only a missing subset of those three indexes, so an interrupted SQLite repair is resumable; it backs up SQLite, validates newly historical PostgreSQL `0005` metadata, rejects multiple active policies, and continues to reject every other schema drift. - Prevented external `LogRecord` extras from injecting allowlisted request/run fields, rejected FIFO/non-regular archive inputs without blocking by opening them nonblocking before `fstat`, and made oversized audit archives fail on their global line cap before decoding attacker-controlled lines. - Made importer post-commit verification and no-op retention post-verification fail closed instead of treating an unchanged row count as sufficient evidence; PostgreSQL retention paths now preserve their advisory/row-lock gate through the mutation decision. - Made materialized governance scope/minute counters fail closed when they diverge from the never-delete attempt ledger; low or high counter drift cannot be used to bypass admission, mutation, reconciliation, or importer preflight. diff --git a/README.md b/README.md index 6cd6a99..4fa7c03 100644 --- a/README.md +++ b/README.md @@ -373,7 +373,7 @@ uv run python -m app.db.import_sqlite \ | `3` | `committed_but_verification_failed` | 目标已提交完整 precommit 快照，但 postcommit 验证或报告失败；停止服务并独立对账 | | `4` | `commit_outcome_unknown` | PostgreSQL 未确认 COMMIT；原子事务意味着目标可能为空，也可能已完整提交 | -退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、13 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。正常使用后的 `20260828_0005` 不能在 `worker_processes` 有事实时原地降级；先停止 Worker、保存必要事实并显式清空该表后，才能进入 `0005 → 0004`，而 `0004` 原有 ledger/audit downgrade guard 仍继续生效。只有隔离空库用于完整降级/升级往返，处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 +退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、13 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。当前 repair head `20260829_0006` 降到 `0005` 不删除对象；继续跨过 `20260828_0005` 时，只要 `worker_processes` 有事实就会拒绝。先停止 Worker、保存必要事实并显式清空该表后，才能进入 `0005 → 0004`，而 `0004` 原有 ledger/audit downgrade guard 仍继续生效。只有隔离空库用于完整降级/升级往返，处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 ## Audit retention 维护 diff --git a/backend/alembic/versions/20260829_0006_repair_governance_indexes.py b/backend/alembic/versions/20260829_0006_repair_governance_indexes.py new file mode 100644 index 0000000..dc3f5c2 --- /dev/null +++ b/backend/alembic/versions/20260829_0006_repair_governance_indexes.py @@ -0,0 +1,125 @@ +"""Repair three governance indexes missing from an early revision 0004 schema. + +Revision ID: 20260829_0006 +Revises: 20260828_0005 +Create Date: 2026-08-29 00:00:00 UTC +""" + +from collections.abc import Sequence +from typing import Any + +import sqlalchemy as sa + +from alembic import op + +revision: str = "20260829_0006" +down_revision: str | None = "20260828_0005" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _normalized_predicate(value: object) -> str: + predicate = " ".join(str(value).strip().lower().split()) + while predicate.startswith("(") and predicate.endswith(")"): + predicate = predicate[1:-1].strip() + return predicate + + +def _ensure_index( + table_name: str, + index_name: str, + columns: tuple[str, ...], + *, + unique: bool, + sqlite_where: sa.TextClause | None = None, + postgresql_where: sa.TextClause | None = None, +) -> None: + connection = op.get_bind() + existing_indexes: dict[str, dict[str, Any]] = { + str(index["name"]): index for index in sa.inspect(connection).get_indexes(table_name) + } + existing = existing_indexes.get(index_name) + if existing is None: + op.create_index( + index_name, + table_name, + list(columns), + unique=unique, + sqlite_where=sqlite_where, + postgresql_where=postgresql_where, + ) + return + + actual_columns = tuple(str(column) for column in existing.get("column_names", ())) + if actual_columns != columns or bool(existing.get("unique")) is not unique: + raise RuntimeError( + f"Existing index {index_name} does not match the repair migration definition" + ) + + expected_predicate: sa.TextClause | None = None + reflected_predicate: object | None = None + dialect_options = existing.get("dialect_options", {}) + if connection.dialect.name == "sqlite": + expected_predicate = sqlite_where + reflected_predicate = dialect_options.get("sqlite_where") + elif connection.dialect.name == "postgresql": + expected_predicate = postgresql_where + reflected_predicate = dialect_options.get("postgresql_where") + predicate_mismatch = (expected_predicate is None) != (reflected_predicate is None) + if expected_predicate is not None and reflected_predicate is not None: + predicate_mismatch = predicate_mismatch or ( + _normalized_predicate(reflected_predicate) != _normalized_predicate(expected_predicate) + ) + if predicate_mismatch: + raise RuntimeError( + f"Existing index {index_name} does not match the repair migration predicate" + ) + + +def _ensure_active_policy_is_unique() -> None: + policies = sa.table( + "governance_policies", + sa.column("is_active", sa.Boolean()), + ) + active_policy_count = ( + op.get_bind() + .execute( + sa.select(sa.func.count()).select_from(policies).where(policies.c.is_active.is_(True)) + ) + .scalar_one() + ) + if active_policy_count > 1: + raise RuntimeError("Cannot repair governance indexes while multiple active policies exist") + + +def upgrade() -> None: + # This data guard must run before the first repair DDL. In particular, a + # direct Alembic invocation must not leave the two non-unique indexes behind + # when restoring the partial unique policy index would be lossy. + _ensure_active_policy_is_unique() + _ensure_index( + "evaluation_runs", + "ix_evaluation_runs_started_at_id", + ("started_at", "id"), + unique=False, + ) + _ensure_index( + "evaluation_runs", + "ix_evaluation_runs_finished_at_id", + ("finished_at", "id"), + unique=False, + ) + _ensure_index( + "governance_policies", + "uq_governance_policies_single_active", + ("is_active",), + unique=True, + sqlite_where=sa.text("is_active = 1"), + postgresql_where=sa.text("is_active IS TRUE"), + ) + + +def downgrade() -> None: + """Retain indexes that are already part of the canonical revision 0005 schema.""" + + return None diff --git a/backend/app/db/prepare_migrations.py b/backend/app/db/prepare_migrations.py index cd9a6d2..77093b1 100644 --- a/backend/app/db/prepare_migrations.py +++ b/backend/app/db/prepare_migrations.py @@ -31,6 +31,7 @@ RELIABILITY_REVISION = "20260825_0002" CREDENTIAL_REVISION = "20260827_0003" GOVERNANCE_REVISION = "20260827_0004" +WORKER_PROGRESS_REVISION = "20260828_0005" _WORKER_PROGRESS_TABLES = {"worker_processes"} _WORKER_PROGRESS_INDEXES = { @@ -47,6 +48,21 @@ "ix_audit_events_occurred_id", } +_KNOWN_GOVERNANCE_MISSING_INDEX_DIFFERENCE_BY_NAME = { + "ix_evaluation_runs_finished_at_id": ( + "add_index:evaluation_runs.ix_evaluation_runs_finished_at_id" + ), + "ix_evaluation_runs_started_at_id": ( + "add_index:evaluation_runs.ix_evaluation_runs_started_at_id" + ), + "uq_governance_policies_single_active": ( + "add_index:governance_policies.uq_governance_policies_single_active" + ), +} +_KNOWN_GOVERNANCE_MISSING_INDEX_DIFFERENCES = frozenset( + _KNOWN_GOVERNANCE_MISSING_INDEX_DIFFERENCE_BY_NAME.values() +) + _WEB_CREDENTIAL_DIFFERENCES = { "add_table:model_credentials", "add_column:models.credential_source", @@ -476,7 +492,12 @@ def _validate_sqlite_table_options(connection: Connection, *, schema_revision: s ) -def _validate_relational_structure(connection: Connection, *, schema_revision: str) -> None: +def _validate_relational_structure( + connection: Connection, + *, + schema_revision: str, + expected_missing_index_names: frozenset[str] = frozenset(), +) -> None: inspector = sa.inspect(connection) legacy = schema_revision == LEGACY_REVISION pre_reliability = schema_revision in {LEGACY_REVISION, PHASE_1_REVISION} @@ -602,6 +623,7 @@ def _validate_relational_structure(connection: Connection, *, schema_revision: s and table_name == "audit_events" and str(index.name) in _WORKER_PROGRESS_EXISTING_INDEX_NAMES ) + and str(index.name) not in expected_missing_index_names } actual_indexes = { ( @@ -719,10 +741,19 @@ def _validate_check_constraints(connection: Connection, *, schema_revision: str) ) -def _validate_sqlite_database(connection: Connection, *, schema_revision: str) -> None: +def _validate_sqlite_database( + connection: Connection, + *, + schema_revision: str, + expected_missing_index_names: frozenset[str] = frozenset(), +) -> None: _validate_sqlite_ddl_modifiers(connection, schema_revision=schema_revision) _validate_sqlite_table_options(connection, schema_revision=schema_revision) - _validate_relational_structure(connection, schema_revision=schema_revision) + _validate_relational_structure( + connection, + schema_revision=schema_revision, + expected_missing_index_names=expected_missing_index_names, + ) _validate_check_constraints(connection, schema_revision=schema_revision) triggers = connection.exec_driver_sql( "SELECT name, tbl_name FROM sqlite_master WHERE type = 'trigger'" @@ -772,6 +803,21 @@ def validate_sqlite_schema_fingerprint( _validate_sqlite_database(connection, schema_revision=schema_revision) +def _validate_known_governance_index_gap_rows(connection: Connection) -> None: + policies = sa.table( + "governance_policies", + sa.column("is_active", sa.Boolean()), + ) + active_policy_count = connection.execute( + sa.select(sa.func.count()).select_from(policies).where(policies.c.is_active.is_(True)) + ).scalar_one() + if active_policy_count > 1: + raise SchemaPreparationError( + "The known governance index compatibility schema contains multiple active " + "policies; the missing unique index cannot be restored safely." + ) + + def _sqlite_database_path(database_url: str) -> Path: url = make_url(database_url) if url.get_backend_name() != "sqlite": @@ -825,6 +871,7 @@ def prepare_database( RELIABILITY_REVISION, CREDENTIAL_REVISION, GOVERNANCE_REVISION, + WORKER_PROGRESS_REVISION, ) historical_heads = {(revision,) for revision in historical_revisions} @@ -871,29 +918,55 @@ def prepare_database( target_revision: str | None source_revision: str if current_heads in historical_heads: - if not sqlite_locked: - return PreparationResult(action="versioned") source_revision = current_heads[0] - if source_revision == LEGACY_REVISION: - expected_differences = _LEGACY_DIFFERENCES - elif source_revision == PHASE_1_REVISION: - expected_differences = _PHASE_1_DIFFERENCES - elif source_revision == RELIABILITY_REVISION: - expected_differences = _RELIABILITY_DIFFERENCES - else: - expected_differences = ( - _CREDENTIAL_DIFFERENCES - if source_revision == CREDENTIAL_REVISION - else _WORKER_PROGRESS_DIFFERENCES - ) + if not sqlite_locked and source_revision != WORKER_PROGRESS_REVISION: + return PreparationResult(action="versioned") + expected_differences = { + LEGACY_REVISION: _LEGACY_DIFFERENCES, + PHASE_1_REVISION: _PHASE_1_DIFFERENCES, + RELIABILITY_REVISION: _RELIABILITY_DIFFERENCES, + CREDENTIAL_REVISION: _CREDENTIAL_DIFFERENCES, + GOVERNANCE_REVISION: _WORKER_PROGRESS_DIFFERENCES, + WORKER_PROGRESS_REVISION: (), + }[source_revision] differences = _schema_differences(connection) - if differences != expected_differences: + expected_missing_index_names: frozenset[str] = frozenset() + expected_difference_set = set(expected_differences) + difference_set = set(differences) + missing_index_differences = difference_set - expected_difference_set + known_governance_index_gap = ( + source_revision + in { + GOVERNANCE_REVISION, + WORKER_PROGRESS_REVISION, + } + and bool(missing_index_differences) + and len(difference_set) == len(differences) + and expected_difference_set <= difference_set + and missing_index_differences <= _KNOWN_GOVERNANCE_MISSING_INDEX_DIFFERENCES + ) + if differences != expected_differences and not known_governance_index_gap: rendered_differences = ", ".join(sorted(differences)) or "unknown" raise SchemaPreparationError( "The versioned historical database does not match its expected schema; " f"upgrade was not started. Differences: {rendered_differences}" ) - _validate_sqlite_database(connection, schema_revision=source_revision) + if known_governance_index_gap: + expected_missing_index_names = frozenset( + index_name + for index_name, difference in ( + _KNOWN_GOVERNANCE_MISSING_INDEX_DIFFERENCE_BY_NAME.items() + ) + if difference in missing_index_differences + ) + _validate_known_governance_index_gap_rows(connection) + if not sqlite_locked: + return PreparationResult(action="versioned") + _validate_sqlite_database( + connection, + schema_revision=source_revision, + expected_missing_index_names=expected_missing_index_names, + ) target_revision = None action = { LEGACY_REVISION: "versioned_legacy", @@ -901,6 +974,7 @@ def prepare_database( RELIABILITY_REVISION: "versioned_reliability", CREDENTIAL_REVISION: "versioned_credentials", GOVERNANCE_REVISION: "versioned_governance", + WORKER_PROGRESS_REVISION: "versioned_worker_progress", }[source_revision] else: if not sqlite_locked: @@ -1027,6 +1101,11 @@ def main() -> int: "Verified a versioned governance/audit SQLite schema before upgrade; " f"backup: {result.backup_path}" ) + elif result.action == "versioned_worker_progress": + print( + "Verified a versioned Worker-progress SQLite schema before upgrade; " + f"backup: {result.backup_path}" + ) return 0 diff --git a/backend/app/governance/audit_archive.py b/backend/app/governance/audit_archive.py index e7bcdea..7d407d5 100644 --- a/backend/app/governance/audit_archive.py +++ b/backend/app/governance/audit_archive.py @@ -30,7 +30,12 @@ from app.models import AuditRetentionClass ARCHIVE_SCHEMA = "llmbenchlab-audit-archive-v1" -ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS = frozenset({"20260828_0005"}) +ARCHIVE_V1_COMPATIBLE_ALEMBIC_HEADS = frozenset( + { + "20260828_0005", + "20260829_0006", + } +) ARCHIVE_V1_RETENTION_VALUES = ("operational", "security") ARCHIVE_EVENT_LIMIT = 10_000 ARCHIVE_FILE_LIMIT_BYTES = 128 * 1024 * 1024 diff --git a/backend/tests/test_audit_archive.py b/backend/tests/test_audit_archive.py index 33abcca..5cc71d7 100644 --- a/backend/tests/test_audit_archive.py +++ b/backend/tests/test_audit_archive.py @@ -282,6 +282,17 @@ def test_archive_v1_rejects_unsupported_source_revision_on_write_and_read(tmp_pa verify_archive(path) +def test_archive_v1_accepts_schema_equivalent_repair_revision() -> None: + data, _digest = build_archive_bytes( + (), + cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), + source_alembic_head="20260829_0006", + has_more_eligible=False, + ) + + assert b'"source_alembic_head":"20260829_0006"' in data + + def test_frozen_archive_v1_fixture_remains_verifiable(tmp_path) -> None: fixture = Path(__file__).with_name("fixtures") / "audit-archive-v1-empty.jsonl" path = tmp_path / "archive-v1-empty.jsonl" diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py index 011e3cd..3477054 100644 --- a/backend/tests/test_migrations.py +++ b/backend/tests/test_migrations.py @@ -7,15 +7,18 @@ import subprocess import sys from pathlib import Path +from unittest.mock import Mock import pytest +import app.db.prepare_migrations as prepare_migrations_module from app.db.base import Base from app.db.prepare_migrations import ( CREDENTIAL_REVISION, GOVERNANCE_REVISION, LEGACY_REVISION, PHASE_1_REVISION, + WORKER_PROGRESS_REVISION, SchemaPreparationError, database_heads, prepare_database, @@ -25,7 +28,7 @@ from app.db.session import create_database_engine BACKEND_ROOT = Path(__file__).resolve().parents[1] -HEAD_REVISION = "20260828_0005" +HEAD_REVISION = "20260829_0006" WEB_CREDENTIAL_REVISION = CREDENTIAL_REVISION RELIABILITY_REVISION = "20260825_0002" @@ -122,6 +125,66 @@ def _insert_legacy_rows(database_path: Path) -> None: ) +def _drop_known_governance_indexes(database_path: Path) -> None: + with sqlite3.connect(database_path) as connection: + for index_name in ( + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + ): + connection.execute(f'DROP INDEX "{index_name}"') + + +def _insert_active_policy( + connection: sqlite3.Connection, + *, + policy_id: str, + version: int, +) -> None: + connection.execute( + "INSERT INTO governance_policies (" + "id, version, policy_hash, is_active, backlog_limit, question_quantum, " + "activated_at, created_at" + ") VALUES (?, ?, ?, 1, 100, 25, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + (policy_id, version, f"{version:064x}"), + ) + + +def _mock_postgresql_0005_preflight( + monkeypatch: pytest.MonkeyPatch, + *, + differences: tuple[str, ...], +) -> tuple[Mock, Mock, list[object]]: + connection = Mock() + connection.dialect.name = "postgresql" + connection.in_transaction.return_value = False + connection.execute.return_value.scalar_one.return_value = 0 + engine = Mock() + engine.connect.return_value = connection + inspector = Mock() + inspector.get_table_names.return_value = list(Base.metadata.tables) + metadata_calls: list[object] = [] + + def schema_differences(candidate: object) -> tuple[str, ...]: + metadata_calls.append(candidate) + return differences + + monkeypatch.setattr(prepare_migrations_module, "create_database_engine", lambda _url: engine) + monkeypatch.setattr( + prepare_migrations_module, + "database_heads", + lambda _connection: (WORKER_PROGRESS_REVISION,), + ) + monkeypatch.setattr( + prepare_migrations_module, + "expected_database_heads", + lambda _config_path: (HEAD_REVISION,), + ) + monkeypatch.setattr(prepare_migrations_module.sa, "inspect", lambda _connection: inspector) + monkeypatch.setattr(prepare_migrations_module, "_schema_differences", schema_differences) + return engine, connection, metadata_calls + + def _rewrite_table_ddl(database_path: Path, table_name: str, old: str, new: str) -> None: """Inject one valid SQLite schema drift without rebuilding unrelated tables.""" @@ -421,6 +484,307 @@ def test_prepare_backs_up_and_adopts_governance_schema( _run_alembic(database_path, "check") +def test_known_governance_index_gap_upgrades_through_0005_and_0006( + tmp_path: Path, +) -> None: + database_path = tmp_path / "known-governance-index-gap.db" + _run_alembic(database_path, "upgrade", LEGACY_REVISION) + _insert_legacy_rows(database_path) + _run_alembic(database_path, "upgrade", GOVERNANCE_REVISION) + with sqlite3.connect(database_path) as connection: + _insert_active_policy(connection, policy_id="policy-active", version=1) + data_counts_before = connection.execute( + "SELECT " + "(SELECT COUNT(*) FROM models), " + "(SELECT COUNT(*) FROM benchmarks), " + "(SELECT COUNT(*) FROM questions), " + "(SELECT COUNT(*) FROM evaluation_runs), " + "(SELECT COUNT(*) FROM evaluation_responses), " + "(SELECT COUNT(*) FROM governance_policies)" + ).fetchone() + _drop_known_governance_indexes(database_path) + + governance_result = prepare_database(_database_url(database_path)) + + assert governance_result.action == "versioned_governance" + assert governance_result.backup_path is not None + assert governance_result.backup_path.is_file() + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) + with sqlite3.connect(database_path) as connection: + indexes = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert ( + not { + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + } + & indexes + ) + + _run_alembic(database_path, "upgrade", WORKER_PROGRESS_REVISION) + + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) + with sqlite3.connect(database_path) as connection: + tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + indexes = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert "worker_processes" in tables + assert { + "ix_worker_processes_stopped_seen_generation", + "ix_audit_events_expires_id", + "ix_audit_events_occurred_id", + } <= indexes + assert ( + not { + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + } + & indexes + ) + + worker_progress_result = prepare_database(_database_url(database_path)) + + assert worker_progress_result.action == "versioned_worker_progress" + assert worker_progress_result.backup_path is not None + assert worker_progress_result.backup_path.is_file() + _run_alembic(database_path, "upgrade", "head") + + assert _read_heads(database_path) == (HEAD_REVISION,) + with sqlite3.connect(database_path) as connection: + data_counts_after = connection.execute( + "SELECT " + "(SELECT COUNT(*) FROM models), " + "(SELECT COUNT(*) FROM benchmarks), " + "(SELECT COUNT(*) FROM questions), " + "(SELECT COUNT(*) FROM evaluation_runs), " + "(SELECT COUNT(*) FROM evaluation_responses), " + "(SELECT COUNT(*) FROM governance_policies)" + ).fetchone() + assert data_counts_after == data_counts_before + for index_name, expected_columns in ( + ("ix_evaluation_runs_started_at_id", ["started_at", "id"]), + ("ix_evaluation_runs_finished_at_id", ["finished_at", "id"]), + ): + assert [ + row[2] for row in connection.execute(f'PRAGMA index_info("{index_name}")') + ] == expected_columns + active_index = { + row[1]: row for row in connection.execute("PRAGMA index_list(governance_policies)") + }["uq_governance_policies_single_active"] + assert active_index[2] == 1 + assert active_index[4] == 1 + active_index_sql = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='index' " + "AND name='uq_governance_policies_single_active'" + ).fetchone()[0] + assert active_index_sql.endswith("WHERE is_active = 1") + with pytest.raises(sqlite3.IntegrityError): + _insert_active_policy(connection, policy_id="policy-active-second", version=2) + connection.rollback() + _run_alembic(database_path, "check") + + _run_alembic(database_path, "downgrade", WORKER_PROGRESS_REVISION) + + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) + with sqlite3.connect(database_path) as connection: + indexes = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert { + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + } <= indexes + canonical_0005_result = prepare_database(_database_url(database_path)) + assert canonical_0005_result.action == "versioned_worker_progress" + _run_alembic(database_path, "upgrade", "head") + assert _read_heads(database_path) == (HEAD_REVISION,) + + +def test_known_governance_index_gap_resumes_after_partial_0006_ddl(tmp_path: Path) -> None: + database_path = tmp_path / "partial-governance-index-repair.db" + _run_alembic(database_path, "upgrade", WORKER_PROGRESS_REVISION) + _drop_known_governance_indexes(database_path) + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE INDEX ix_evaluation_runs_started_at_id ON evaluation_runs (started_at, id)" + ) + + result = prepare_database(_database_url(database_path)) + + assert result.action == "versioned_worker_progress" + assert result.backup_path is not None and result.backup_path.is_file() + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) + _run_alembic(database_path, "upgrade", "head") + + assert _read_heads(database_path) == (HEAD_REVISION,) + with sqlite3.connect(database_path) as connection: + indexes = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert { + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + } <= indexes + _run_alembic(database_path, "check") + + +@pytest.mark.parametrize( + ("differences", "expected_row_guard_calls"), + [ + ( + (), + 0, + ), + ( + ("add_index:evaluation_runs.ix_evaluation_runs_started_at_id",), + 1, + ), + ( + ( + "add_index:evaluation_runs.ix_evaluation_runs_finished_at_id", + "add_index:evaluation_runs.ix_evaluation_runs_started_at_id", + "add_index:governance_policies.uq_governance_policies_single_active", + ), + 1, + ), + ], +) +def test_prepare_postgresql_0005_accepts_only_canonical_or_known_index_gap( + monkeypatch: pytest.MonkeyPatch, + differences: tuple[str, ...], + expected_row_guard_calls: int, +) -> None: + engine, connection, metadata_calls = _mock_postgresql_0005_preflight( + monkeypatch, + differences=differences, + ) + + result = prepare_database("postgresql+psycopg://localhost/llmbenchlab_test") + + assert result.action == "versioned" + assert metadata_calls == [connection] + assert connection.execute.call_count == expected_row_guard_calls + connection.close.assert_called_once_with() + engine.dispose.assert_called_once_with() + + +def test_prepare_postgresql_0005_rejects_unknown_drift_with_known_index_gap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, connection, metadata_calls = _mock_postgresql_0005_preflight( + monkeypatch, + differences=( + "add_index:evaluation_runs.ix_evaluation_runs_started_at_id", + "add_index:models.ix_models_enabled", + ), + ) + + with pytest.raises(SchemaPreparationError, match="historical database"): + prepare_database("postgresql+psycopg://localhost/llmbenchlab_test") + + assert metadata_calls == [connection] + connection.execute.assert_not_called() + connection.close.assert_called_once_with() + engine.dispose.assert_called_once_with() + + +def test_known_governance_index_gap_rejects_duplicate_active_policies( + tmp_path: Path, +) -> None: + database_path = tmp_path / "duplicate-active-policy-index-gap.db" + _run_alembic(database_path, "upgrade", GOVERNANCE_REVISION) + _drop_known_governance_indexes(database_path) + with sqlite3.connect(database_path) as connection: + _insert_active_policy(connection, policy_id="policy-active-first", version=1) + _insert_active_policy(connection, policy_id="policy-active-second", version=2) + + with pytest.raises(SchemaPreparationError, match="multiple active policies"): + prepare_database(_database_url(database_path)) + + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) + assert list(tmp_path.glob("*.bak")) == [] + + +def test_governance_index_repair_migration_guards_duplicate_active_policies_before_ddl( + tmp_path: Path, +) -> None: + database_path = tmp_path / "direct-upgrade-duplicate-active-policy.db" + _run_alembic(database_path, "upgrade", WORKER_PROGRESS_REVISION) + _drop_known_governance_indexes(database_path) + with sqlite3.connect(database_path) as connection: + _insert_active_policy(connection, policy_id="policy-active-first", version=1) + _insert_active_policy(connection, policy_id="policy-active-second", version=2) + + failed = _invoke_alembic(database_path, "upgrade", "head") + + assert failed.returncode != 0 + assert "multiple active policies exist" in failed.stderr + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) + with sqlite3.connect(database_path) as connection: + indexes = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='index'") + } + assert ( + not { + "ix_evaluation_runs_started_at_id", + "ix_evaluation_runs_finished_at_id", + "uq_governance_policies_single_active", + } + & indexes + ) + + +def test_governance_index_repair_rejects_same_name_partial_run_index(tmp_path: Path) -> None: + database_path = tmp_path / "direct-upgrade-partial-run-index.db" + _run_alembic(database_path, "upgrade", WORKER_PROGRESS_REVISION) + with sqlite3.connect(database_path) as connection: + connection.execute("DROP INDEX ix_evaluation_runs_started_at_id") + connection.execute( + "CREATE INDEX ix_evaluation_runs_started_at_id " + "ON evaluation_runs (started_at, id) WHERE started_at IS NOT NULL" + ) + + failed = _invoke_alembic(database_path, "upgrade", "head") + + assert failed.returncode != 0 + assert "does not match the repair migration predicate" in failed.stderr + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) + with sqlite3.connect(database_path) as connection: + index_sql = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='index' " + "AND name='ix_evaluation_runs_started_at_id'" + ).fetchone()[0] + assert index_sql.endswith("WHERE started_at IS NOT NULL") + + +def test_known_governance_index_gap_does_not_allow_additional_drift(tmp_path: Path) -> None: + database_path = tmp_path / "governance-index-gap-with-extra-drift.db" + _run_alembic(database_path, "upgrade", GOVERNANCE_REVISION) + _drop_known_governance_indexes(database_path) + with sqlite3.connect(database_path) as connection: + connection.execute("DROP INDEX ix_models_enabled") + + with pytest.raises(SchemaPreparationError, match="historical database"): + prepare_database(_database_url(database_path)) + + assert _read_heads(database_path) == (GOVERNANCE_REVISION,) + assert list(tmp_path.glob("*.bak")) == [] + + @pytest.mark.parametrize( ("cancellation_requested", "expected_status", "expected_last_error"), [ @@ -801,7 +1165,9 @@ def test_worker_progress_downgrade_rejects_process_facts_before_ddl(tmp_path: Pa assert failed.returncode != 0 assert "process facts exist" in failed.stderr - assert _read_heads(database_path) == (HEAD_REVISION,) + # Revision 0006 has a schema-no-op downgrade, so the 0005 evidence guard + # still refuses before any Worker table or index DDL is attempted. + assert _read_heads(database_path) == (WORKER_PROGRESS_REVISION,) with sqlite3.connect(database_path) as connection: assert connection.execute("SELECT COUNT(*) FROM worker_processes").fetchone()[0] == 1 audit_indexes = {row[1] for row in connection.execute("PRAGMA index_list(audit_events)")} diff --git a/backend/tests/test_phase2_acceptance_script.py b/backend/tests/test_phase2_acceptance_script.py index 7df2d19..90b249a 100644 --- a/backend/tests/test_phase2_acceptance_script.py +++ b/backend/tests/test_phase2_acceptance_script.py @@ -34,6 +34,7 @@ def test_migration_acceptance_retains_both_populated_guards_and_empty_round_trip assert script.PRE_GOVERNANCE_REVISION == "20260827_0003" assert script.GOVERNANCE_REVISION == "20260827_0004" assert script.WORKER_PROGRESS_REVISION == "20260828_0005" + assert script.DATABASE_HEAD_REVISION == "20260829_0006" assert "postgres_populated_0005_and_0004_downgrade_guards_with_empty_round_trips" in source assert "Cannot downgrade Worker progress schema" in source assert "Cannot downgrade governance schema" in source diff --git a/backend/tests/test_phase2_capacity_script.py b/backend/tests/test_phase2_capacity_script.py index d772199..6fa2dcf 100644 --- a/backend/tests/test_phase2_capacity_script.py +++ b/backend/tests/test_phase2_capacity_script.py @@ -1974,6 +1974,7 @@ def test_acceptance_requires_both_populated_guards_and_empty_round_trips() -> No assert 'PRE_GOVERNANCE_REVISION = "20260827_0003"' in source assert 'GOVERNANCE_REVISION = "20260827_0004"' in source assert 'WORKER_PROGRESS_REVISION = "20260828_0005"' in source + assert 'DATABASE_HEAD_REVISION = "20260829_0006"' in source assert "application_worker_progress_0005_to_0004_guard" in source assert "isolated_governance_0004_to_0003_guard" in source assert "worker_progress_0005_to_0004_round_trip" in source diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c6318e8..5a77147 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -560,7 +560,7 @@ React 主导航包含独立的 Runs 列表页。该页通过 `GET /runs` 以 20 带凭据的目标 DSN 必须通过 `--target-env` 从受控环境读取；`--target` 拒绝 URL password 和 password query。COMMIT 未获得 PostgreSQL 确认时退出 `4`/`commit_outcome_unknown`；由于事务原子性，目标可能为空，也可能是完整的 precommit 快照。COMMIT 已确认但连接收尾、postcommit 快照/对账或报告失败时退出 `3`/`committed_but_verification_failed`；这时目标已提交完整 precommit 快照，不会自动回滚。两种结果都禁止盲目重试，必须保持目标离线，按已输出摘要独立检查目标是空还是完整提交。非空目标会拒绝再次导入，工具也不提供 PostgreSQL 到 SQLite 的反向同步。 -Alembic `20260827_0004` 引入治理/审计表与 Run/Response 字段；`20260828_0005` 增加 `worker_processes` 与 audit retention/exporter 扫描索引。两个 downgrade guard 都在第一条 DDL 前拒绝可能丢失的事实：0005 拒绝任意 Worker generation，0004 拒绝 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 证据。隔离空数据库分别验证 `0005 ↔ 0004` 与 `0004 ↔ 0003`；已使用环境应优先向前修复，或恢复经核验的旧备份并单独保留新 schema 证据。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 +Alembic `20260827_0004` 引入治理/审计表与 Run/Response 字段；`20260828_0005` 增加 `worker_processes` 与 audit retention/exporter 扫描索引；schema-equivalent `20260829_0006` 只条件补齐早期 `0004` 变体缺少的三个 canonical 索引。兼容 preflight 仍以精确 fingerprint、integrity/FK、索引定义和 single-active 数据约束 fail closed。`0006 → 0005` 不删除 canonical 对象；后续两个 downgrade guard 都在第一条有损 DDL 前拒绝可能丢失的事实：0005 拒绝任意 Worker generation，0004 拒绝 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 证据。隔离空数据库分别验证 `0005 ↔ 0004` 与 `0004 ↔ 0003`；已使用环境应优先向前修复，或恢复经核验的旧备份并单独保留新 schema 证据。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 ## 错误处理与可观察性 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 641a52d..9547083 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -325,6 +325,8 @@ Redis 开启 AOF (`appendfsync everysec`) 只改善通知持久性，不是备 - `20260825_0002`：attempt、租约、心跳、backoff、queue audit 与 dead-letter 字段/约束/索引。 - `20260827_0003`：Model credential source、AES-GCM `model_credentials` 与 Web write-only Key。 - `20260827_0004`：版本化治理 policy、四层 scope/minute counter、question execution、Provider attempt ledger、typed audit、Run fairness/backpressure 字段和 Response Provider metadata。 +- `20260828_0005`：Worker progress facts 与 audit retention/exporter 有界扫描索引。 +- `20260829_0006`：不改变逻辑 schema 的兼容修复；仅为早期 `0004` 历史变体补齐三个 canonical governance 索引。 本地 SQLite 更新： @@ -332,7 +334,7 @@ Redis 开启 AOF (`appendfsync everysec`) 只改善通知持久性，不是备 make migrate ``` -命令先执行 `app.db.prepare_migrations`，再 `alembic upgrade head`。受支持的未版本化 SQLite 会在严格结构/integrity/FK 检查和一致性备份后 stamp；未知 drift 在写 revision 前拒绝。普通 API/Worker 启动只检查 head，不运行 `create_all`、preflight 或 upgrade。 +命令先执行 `app.db.prepare_migrations`，再 `alembic upgrade head`。受支持的未版本化 SQLite 会在严格结构/integrity/FK 检查和一致性备份后 stamp；已知早期 `0004/0005` 变体只有在 revision fingerprint 为 canonical，或仅缺这三个已知索引的非空子集且最多一条 active policy 时，才会备份并交给 `0006` 修复，因此修复 DDL 中断后可安全重入。新近成为 historical 的 PostgreSQL `0005` 同样先做 metadata drift 校验。未知 drift 在写 revision 或 repair DDL 前拒绝。普通 API/Worker 启动只检查 head，不运行 `create_all`、preflight 或 upgrade。 Compose 中只有一次性 `migrate` 服务执行： @@ -342,7 +344,7 @@ python -m app.db.prepare_migrations && alembic upgrade head && alembic check `api` 与 `worker` 必须等待 migrate exit 0，然后仅执行 head check。不要把 Alembic 命令加回 API/Worker entrypoint，也不要同时运行多个 migration owner。 -`0005 -> 0004` 在 `worker_processes` 有任意 generation fact 时于第一条 DDL 前拒绝；`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时同样拒绝。正常使用后的数据库不能把它们当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0005/0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 +`0006 -> 0005` 是 no-op downgrade，因为三个索引本来就属于 canonical `0004`；`0005 -> 0004` 在 `worker_processes` 有任意 generation fact 时于第一条 DDL 前拒绝；`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时同样拒绝。正常使用后的数据库不能把它们当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0006/0005/0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 ### 5.3 备份与恢复证据边界 diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 5747e2b..3a83317 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -4,7 +4,7 @@ > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) > 当前计划：[Phase 2 恢复与运维闭环](plans/2026-08-28-phase-2-recovery-operations.md) > 当前日志：[2026-08-28 P2-07 工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md) -> 当前决策：[ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) +> 当前决策：[ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)，exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) > 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) ## 当前事实 @@ -13,14 +13,16 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 当前 P2-06 已按 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 实现： -- Alembic head `20260828_0005` 新增 `worker_processes` 和 bounded audit scan indexes；SQLite→PostgreSQL importer 扩展为 13 表精确 digest，拒绝 live generation 并复制 stopped/stale facts；表非空时 `0005 -> 0004` 在 DDL 前拒绝。 +- P2-06 的 `20260828_0005` 新增 `worker_processes` 和 bounded audit scan indexes；当前维护 head `20260829_0006` 仅修复早期 `0004` 三索引缺口，不改变 13 表/importer/archive 语义。SQLite→PostgreSQL importer 继续做 13 表精确 digest，拒绝 live generation 并复制 stopped/stale facts；表非空时 `0005 -> 0004` 在 DDL 前拒绝。 - 长运行 Worker 注册唯一 generation，只在真实 scan/claim/lease-heartbeat/progress 后按 DB UTC 合并刷新；`/tasks/metrics` 和 exporter 只公开 expected/registered/live/stalled/shortfall 与聚合时间。dependency probe 固定声明不检查 main-loop progress。 - `GET /api/v1/metrics/prometheus` 输出固定 Prometheus text `0.0.4` gauge，使用一个 DB-time 快照、有界 15 分钟 audit 与 1 小时 latency 窗口、固定 enum label、整次 fail-closed 和每 API 进程 single-flight。 - `deploy/observability/` 提供固定八条告警规则、抓取示例与对应 Operations Runbook；仓库不部署 Prometheus、Alertmanager、OTel 或通知发送器。 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 提供 canonical JSONL v1、严格文件/权限/大小/schema/hash/rollup 校验、真正离线 verify、精确 digest 绑定、默认不删除和 commit-outcome 分类。它不提供签名/WORM，也不替代整库+keyring backup。 - 全生产日志调用受字面量/无格式参数静态门禁；第三方动态消息固定化、结构化数值有限化，raw Uvicorn access handler 关闭。 -P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。默认用户 SQLite 尚未在 head且未擅自迁移。 +`0006` 兼容修复已完成本地 migration/full test/lint/Mock smoke、真实失败备份副本升级和当前重建库 startup/check；精确 SHA 远程 CI 尚待执行，因此当前标记为 `local_verified`，不改变 P2-07 的 `planned` 状态或范围。 + +P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 ## 已完成：建立 P2-07 工作包 @@ -33,7 +35,7 @@ P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/te P2-07 当前为 `planned`、尚未实现。恢复实施时按当前计划从最小只读 verifier 切片开始，之后才依次完成： -- PostgreSQL backup → 空目标 restore → Alembic `20260828_0005` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 +- PostgreSQL backup → 空目标 restore → Alembic `20260829_0006` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 - 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 - Redis 重建/consumer group 恢复、Worker 扩缩、八条告警响应、dead-letter、commit outcome unknown、governance integrity 与 remaining cancel/retry/lease/budget crash matrix 的真实 PostgreSQL/Redis 演练。 - audit archive 作为数据库恢复后的精确校验/补回工具参与演练，但不得把 archive 自身 restore 冒充整库、PITR、RPO/RTO 或 WORM 认证。 diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index ae024a4..0c9ce5e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -368,7 +368,11 @@ llmbenchlab-audit-retention restore \ 退出码 `0` 表示已确认成功，`2` 表示在提交前安全失败，`3` 表示提交已成功但后验核验失败，`4` 表示 commit outcome unknown。遇到 `3` 或 `4` 不得盲目重跑 mutation：保留 archive/count/digest，先执行只读 `reconcile` 判定数据库与 archive 的精确关系，再由操作者决定恢复或删除。Archive 与数据库/keyring 备份是不同资产；archive 不含 credential ciphertext/nonce/keyring，也不能单独恢复 stored Provider Key。 -## 10. 0005 与 0004 安全回滚 +## 10. 0006、0005 与 0004 安全回滚 + +### 10.0 Governance index compatibility repair 0006 + +revision `20260829_0006` 不增加新的业务字段或表，只为曾执行早期 `0004` 变体的数据库条件补齐 `ix_evaluation_runs_started_at_id`、`ix_evaluation_runs_finished_at_id` 与 `uq_governance_policies_single_active`。`make migrate` 只在 revision fingerprint 为 canonical，或缺失项是这三个索引的非空子集时放行，并先创建 SQLite 一致性备份；这允许在 SQLite repair DDL 部分完成但 marker 仍为 `0005` 时安全重入。PostgreSQL `0005` 也必须通过 metadata drift 校验。若有多条 active policy、错误的同名索引或任何额外 drift，必须人工核对，工具不会自动停用 policy。`0006 -> 0005` 保留这三个 canonical `0004` 索引，因此是有意的 no-op downgrade。 ### 10.1 Worker progress / retention revision 0005 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 6473aa3..f3a5fb5 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -1,6 +1,6 @@ # 项目状态 -> 更新时间：2026-08-28（Asia/Shanghai） +> 更新时间：2026-08-30（Asia/Shanghai） ## 当前阶段 @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。当前 [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / `20260829_0006` 兼容修复仅为 `local_verified`，仍位于未提交工作树，尚未 push 且没有精确 SHA 远程 CI；上述 P2-06 历史绿灯不替代本次门禁。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -46,14 +46,21 @@ - [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；实现 SHA `9a20676dcf545040782f04c166205d0043345753` 将 Alembic head 扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 - `GET /api/v1/metrics/prometheus` 已实现固定 Prometheus text `0.0.4` gauge：一个 DB-time 读快照、15 分钟 typed-audit 窗口、1 小时 Run latency、硬读取上限、固定 enum label、整次 fail-closed 与每 API 进程 single-flight。`deploy/observability/` 提供固定八条规则和安全抓取示例；仓库不部署 Prometheus、Alertmanager 或通知发送器。 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 已实现 canonical JSONL v1、严格权限/大小/行/schema/hash/rollup 校验、离线 verify、精确 digest 绑定、默认不删除、双方言事务与 commit outcome 分类。Archive 是敏感运维文件，hash 只用于完整性/绑定，不是签名或 WORM，也不替代 P2-07 的数据库+keyring 备份。 -- importer 当前合同为 `0005` 的 13 表精确 count/PK/content digest；live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 +- P2-06 的 `0005` 将 importer 逻辑合同扩展为 13 表精确 count/PK/content digest；当前 source/target 仍必须位于唯一 current head `0006`，13 表语义不变。live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 - 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 -- 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。Evidence closeout 文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push，精确 SHA run `33165775037` 4/4 成功，P2-06 仓库级收尾完成。默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 +- 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。Evidence closeout 文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push，精确 SHA run `33165775037` 4/4 成功，P2-06 仓库级收尾完成。P2-06 当时默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 + +## 2026-08-29 数据库兼容修复（`local_verified`） + +- 旧库失败不是 SQLite 损坏，而是 revision=`20260827_0004` 的早期结构变体精确缺少三个后来加入 canonical `0004` 的索引；严格 preflight 因 revision/schema 不一致，在 `0005` 执行前按设计拒绝。 +- [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) 已接受；当前 head 为 schema-equivalent `20260829_0006`。preflight 只接受 canonical `0004/0005`，或仅缺一至三个已知 repair 索引的 fingerprint，以支持 SQLite repair 中断重入；PostgreSQL `0005` 的额外 metadata drift 同样拒绝。完整 schema/integrity/FK、索引定义、single-active policy 数据门禁仍 fail closed。`0006` 首条 DDL 前再次拒绝多条 active policy，并条件补建索引；`0006 -> 0005` 保留 canonical 对象。 +- 真实失败备份副本已无损升级到 `0006` 并通过 integrity/FK/Alembic check；当前重建库由标准 preflight 自动备份后从 canonical `0005` 到 `0006`，startup gate、quick check、FK 和 metadata check 全部通过，业务计数与 Worker facts 保持。 +- 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。精确 SHA 远程 CI 尚未执行，因此本维护任务暂不标记 completed，Phase 2/P2-07 状态不变。 ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 -- P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 +- P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 @@ -78,7 +85,8 @@ | P2-06 `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed` | 实现 SHA `9a20676…` 的本地门禁通过；只用 Mock/Stub | | P2-06 Mock smoke | `1 passed, 7 deselected` | 完全离线通过 | | P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；实现 SHA 的远程 integration job 也已成功 | -| P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；默认用户 SQLite 非 head 的 check 失败后未迁移 | current head `20260828_0005`；用户数据保持原状 | +| P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；当时默认用户 SQLite 非 head 的 check 失败后未迁移 | P2-06 当时 head `20260828_0005`；历史证据保持原状 | +| 2026-08-29 DB compatibility repair | migration `52 passed`；最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；本地已验证，exact-SHA CI 待执行 | | P2-06 build/config | frontend build 成功（保留 662.39 kB chunk warning）；Compose config exit 0 | 从根目录误跑 npm 的失败已记录并用正确目录重跑通过 | | P2-06 Prometheus 规则 | `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | 临时容器验证；仓库仍不部署 Prometheus/Alertmanager | | P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`、`0005`/isolated `0004` populated refusal、两层空库往返、cleanup C/V/N empty | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b248738..c671c16 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -288,7 +288,7 @@ Redis 仅可置于受控内部网络。当前本地 Compose 使用 AOF、无 ACL - 退出码 `3` 表示 `COMMIT` 已确认、但提交后验证或报告失败；导入已经提交。**禁止盲目重试或把它描述为回滚**，应先只读核验目标，必要时从已验证备份执行人工恢复。 - 工具是单向导入，不提供 PostgreSQL→SQLite 自动回迁。schema downgrade 也不等于数据平台回滚。 -`20260828_0005 → 0004` 会移除 Worker progress，guard 在第一条 DDL 前拒绝任何 generation 行；`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata并拒绝任何相关事实。正常运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的旧备份并保留只读证据。只有隔离空数据库用于双方向往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 +`20260829_0006` 只补齐 canonical `0004` 的三个索引；兼容入口只接受 canonical schema，或缺失项为这三个索引的非空子集，以便 SQLite 非事务 DDL 中断后安全重入。新近成为 historical 的 PostgreSQL `0005` 也必须先通过 metadata drift 校验；任何额外差异仍 fail closed。SQLite preflight 与 repair migration 都会在恢复 single-active 唯一索引前拒绝多条 active policy。`0006 → 0005` 不删除对象；`20260828_0005 → 0004` 会移除 Worker progress，guard 在第一条 DDL 前拒绝任何 generation 行；`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata并拒绝任何相关事实。正常运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的旧备份并保留只读证据。只有隔离空数据库用于双方向往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 ## 10. 依赖与构建供应链 diff --git a/docs/TESTING.md b/docs/TESTING.md index d6f6685..7827437 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -262,7 +262,7 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 `backend/tests/test_migrations.py` 使用独立临时 SQLite 和 Alembic 子进程验证： -- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260828_0005` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata、`worker_processes` 字段/约束，以及两个 bounded audit scan indexes。 +- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260829_0006` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata、`worker_processes` 字段/约束、两个 bounded audit scan indexes，以及早期 `0004/0005` 三索引缺口、repair DDL 部分完成后的重入和 PostgreSQL `0005` metadata 白名单/额外 drift 拒绝控制流（Mock）。 - 有模型、Benchmark、题目、Run 与 Response 的 legacy schema 被一致性备份、严格识别并无损升级；题目按原插入顺序回填 0-based `position`。 - 与当前 metadata 一致但没有版本标记的库可安全收养，已有 head 重复 preflight 不生成多余备份。 - 部分表、server default/CHECK 内容或重名、PK/UNIQUE/FK/index/partial index、trigger、SQLite conflict policy/generated column、`STRICT`/`WITHOUT ROWID` 等未知 drift 在创建版本标记和备份前被拒绝；已在 head 的库同样验证。 @@ -280,7 +280,7 @@ cd backend uv run pytest tests/test_migrations.py ``` -真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline 并停止 API/Worker 后，先断言 populated `0005 -> 0004` 在任何 DDL 前拒绝且 head 不变；另建隔离空 PostgreSQL 完成 `0005 -> 0004 -> 0005`/check。历史 `0004 -> 0003` governance/audit guard 仍保留；schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 +真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline 并停止 API/Worker 后，从 head `0006` 发起 downgrade；`0006 -> 0005` 为 no-op，随后 populated `0005 -> 0004` 在任何有损 DDL 前拒绝。另建隔离空 PostgreSQL 跨过 `0005 -> 0004 -> 0005`，最终 `upgrade head` 回到 `0006` 并 check。历史 `0004 -> 0003` governance/audit guard 仍保留；schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 ### 6.3 SQLite→PostgreSQL 导入 @@ -397,7 +397,8 @@ P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史 | `make test` | 后端 `916 passed, 33 skipped`；前端 `38 passed`；模型路径只使用 Mock/MockTransport/stub | | `make smoke` | `1 passed, 7 deselected`；完全离线 Mock | | Frontend production build | 从根目录误运行 `npm run build` 因无 `package.json` 失败；改为 `cd frontend && npm run build` 后成功（2192 modules），保留 662.39 kB 主 chunk 非阻断 warning | -| Migration | 默认用户 SQLite 尚未在 head，直接 `alembic check` 失败且未擅自迁移；临时 SQLite head→`0001`→head/check 与隔离真实 PostgreSQL 往返/check 全绿；当前 head `20260828_0005` | +| Migration | 默认用户 SQLite 尚未在 head，直接 `alembic check` 失败且未擅自迁移；临时 SQLite head→`0001`→head/check 与隔离真实 PostgreSQL 往返/check 全绿；P2-06 当时 head `20260828_0005` | +| 2026-08-29 DB compatibility repair（本地） | 真实失败备份副本 `0004 -> 0005 -> 0006`、migration `52 passed`、最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`、当前重建库 migrate/startup/check、lint/smoke/build/config 全绿；current head `20260829_0006`。本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance；historical PG `0005` missing-index 分支仅有 Mock 控制流回归，exact-SHA CI 的真实 PG job 覆盖 fresh canonical 分支 | | Compose config | `docker compose config --quiet` exit 0 | | Prometheus rules | 临时 `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | | Ruff scripts 补充检查 | 对 `scripts/` 的过宽默认 Ruff 命令报告 93 条既有 modernization 告警；按本任务合同使用 `--select E,F,I` 后通过，未把既有告警冒充本次失败 | @@ -574,7 +575,7 @@ make phase2-acceptance 6. Redis 完全 stop/start；`live`/`health` 保持可用、`ready` 降级，API 仍以 `202` 提交数据库事实，Worker 仅靠 DB reconciliation 完成；Redis 恢复后新消息正常 ACK。 7. Worker 停止时取消 pending Run；Worker 恢复消费旧通知后终态和 0 Response 不漂移。 8. 运行中取消并再次 XADD 同一 Run；Response 数在取消后冻结，重复投递被 ACK 且 canonical snapshot 不变。 -9. 停止 API/Worker 后在 populated PostgreSQL 上尝试 `20260828_0005 -> 0004`：Worker progress rows 存在时必须在第一条 DDL 前拒绝，application revision、13 表计数、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0005 -> 0004 -> 0005` 和 check，随后重启 API/Worker。历史 `0004` governance/audit guard 继续由 migration 回归覆盖；schema downgrade 不是数据平台回迁。 +9. 停止 API/Worker 后从 current head `20260829_0006` 尝试 downgrade 到 `0004`：schema-no-op `0006 -> 0005` 后，Worker progress rows 存在时必须在 `0005 -> 0004` 第一条有损 DDL 前拒绝，13 表计数、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0005 -> 0004 -> 0005`，最终回到 `0006` 并 check，随后重启 API/Worker。历史 `0004` governance/audit guard 继续由 migration 回归覆盖；schema downgrade 不是数据平台回迁。 任何一个场景失败、未运行、使用真实 Provider、最终 PEL/lag 非零或清理不完整，都不能把可靠执行基础写成通过。`--self-check-only` 只验证 Docker/Compose、隔离和清理 guard，不执行九场景，不能替代正式命令。精确 SHA `665244e…` 的最终本地运行已 9/9 通过；artifact 与 hash 见第 10.1 节。 diff --git a/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md index 2be8a0c..8a4c327 100644 --- a/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md +++ b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md @@ -5,6 +5,7 @@ - **Deciders**: LLMBenchLab maintainers - **Scope**: Phase 2 P2-06 metrics exporter、告警、Worker 主循环进展与 audit retention - **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的任务可观测性、[ADR-0009](ADR-0009-database-governance-audit-fair-scheduling.md) 的 typed audit 保留流程，以及 [ADR-0010](ADR-0010-phase-2-governance-delivery-boundaries.md) 的恢复指标边界 +- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将 schema-equivalent `0006` 加入 archive-v1 compatible-head allowlist - **Preserves**: PostgreSQL/数据库事实来源、Redis 非权威通知、`llmbenchlab-protocol-v1`、write-only Provider Key、Mock-only 自动化和可信 loopback 部署边界 ## Context @@ -220,7 +221,7 @@ Dead-letter 与 integrity 是 15 分钟 rolling symptom；Prometheus 整个窗 - cutoff 不由用户任意传入；空集合也生成有效 archive。 - archive 默认绝不删除；运维顺序固定为 archive -> offline verify -> maintenance-window delete。 -单文件 canonical JSONL schema 为 `llmbenchlab-audit-archive-v1`：一行 header、零至一万行 `audit_event`、唯一末行 manifest。V1 的 event/payload/retention contract 与 compatible Alembic head allowlist 独立冻结；当前唯一 compatible head 是 `20260828_0005`。`source_alembic_head` 不是装饰性字符串：write/verify/restore/delete/reconcile 都拒绝未列入 allowlist 的旧版、分支或未来 head；未来 schema 只有在证明 V1 全字段语义仍兼容后才能显式扩充 allowlist，否则必须提升 archive schema 并保留 V1 reader。Event 保存完整恢复事实： +单文件 canonical JSONL schema 为 `llmbenchlab-audit-archive-v1`：一行 header、零至一万行 `audit_event`、唯一末行 manifest。V1 的 event/payload/retention contract 与 compatible Alembic head allowlist 独立冻结；compatible heads 为 `20260828_0005` 与 schema-equivalent repair head `20260829_0006`。后者只恢复 canonical `0004` 的三个索引，不改变 archive event、字段或保留语义。`source_alembic_head` 不是装饰性字符串：write/verify/restore/delete/reconcile 都拒绝未列入 allowlist 的旧版、分支或未来 head；未来 schema 只有在证明 V1 全字段语义仍兼容后才能显式扩充 allowlist，否则必须提升 archive schema 并保留 V1 reader。Event 保存完整恢复事实： ```text id, event_key, event_type, payload_hash, payload, retention_class, @@ -278,7 +279,7 @@ Delete 永不执行宽泛 `DELETE WHERE expires_at < cutoff`： ### 11. Schema、migration 与 importer -新 Alembic head `20260828_0005`，同时是 archive-v1 当前唯一 compatible source/target head： +Worker progress revision 为 `20260828_0005`；当前 Alembic head `20260829_0006` 是 schema-equivalent index repair。两者都是 archive-v1 compatible source/target head： - 创建 `worker_processes` 及本文约束/索引； - 为 audit archive 扫描增加 `(expires_at,id)` 索引； @@ -287,6 +288,8 @@ Delete 永不执行宽泛 `DELETE WHERE expires_at < cutoff`： `prepare_migrations.py` 将 `20260827_0004` 纳入 historical head/fingerprint/备份/adoption，严格验证新表、约束和索引。`0005 -> 0004` 在第一条 DDL 前要求 `worker_processes` 为空；有行即拒绝且不得静默丢失事实。空隔离库继续支持 downgrade/re-upgrade。 +`0006` 对三个 canonical governance 索引逐一检查并只补建缺失项；SQLite preflight 仅对白名单 `0004/0005` canonical fingerprint 或这三个索引的缺失子集放行，并在创建 single-active 唯一部分索引前确认 active policy 不超过一条。`0006 -> 0005` 不删除这些本应属于 `0004` 的索引。 + SQLite -> PostgreSQL importer 的 core table list 从 12 增至 13，精确复制 stopped 和已经 stale 的 Worker facts并纳入 count/PK/content digest、目标空表与锁序；source preflight 使用 DB now/stale threshold拒绝仍 live 的 generation，防止把正在运行的 Worker 数据库迁走。Stale crash row不会被当成 live，也不会让 expected shortfall 消失。 Audit archive 文件不是 importer 的第十四张表，也不由 importer 自动读取。P2-07 后续整库恢复必须独立证明 13 表与 keyring 配对；本决定不提前完成该工作。 diff --git a/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md index fd9317d..538eb49 100644 --- a/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md +++ b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md @@ -5,6 +5,7 @@ - **Deciders**: LLMBenchLab maintainers - **Scope**: Phase 2 P2-07 备份/恢复验证、Redis rebuild、告警响应与故障矩阵 - **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的恢复运维边界、[ADR-0007](ADR-0007-web-provider-credentials.md) 的数据库外 keyring 备份边界，以及 [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的告警与 archive 后续演练 +- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将尚未实施的 recovery-manifest-v1 exact head 显式更新为 schema-equivalent `20260829_0006` - **Preserves**: PostgreSQL 唯一任务事实来源、Redis 非权威通知、write-only Provider Key、`llmbenchlab-protocol-v1`、Mock-only 自动化和可信本地运维边界 ## Context @@ -38,7 +39,7 @@ Phase 2 已交付 PostgreSQL 任务/治理事实、Redis Streams 通知、独立 - 创建备份前停止 admission、API、Worker、CLI mutation 和所有 application/audit writer；确认 Worker 已 graceful stop 或进入可解释 stale 状态。数据库健康探针可以继续只读连接，但任何未知 writer 都使备份集不合格。 - `pg_dump` 与 source manifest snapshot 必须来自同一停写窗口。恢复后的 13 表摘要与 manifest 不一致即证明期间发生漂移或 artifact 不匹配，整组 fail closed。 - 目标必须由操作者预先创建且没有任何用户 schema/table/Alembic row。先运行只读 empty-target 检查；目标非空时拒绝，绝不由工具清空。 -- 恢复后数据库必须已经精确位于 Alembic `20260828_0005`。禁止先升级旧 dump 再把它报告为原备份恢复成功；若未来支持新 head，必须在新 ADR/manifest schema 中显式声明兼容。 +- 按 [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md)，恢复后数据库必须已经精确位于 Alembic `20260829_0006`。该 revision 与 `0005` 逻辑 schema 等价，只修复早期 `0004` 三索引缺口；P2-07 尚未实施，因此 recovery-manifest-v1 在实现前直接固定当前 head。禁止先升级旧 dump 再把它报告为原备份恢复成功；若未来支持新 head，必须在新 ADR/manifest schema 中显式声明兼容。 - 13 张核心表固定为： ```text @@ -70,8 +71,8 @@ Manifest schema 固定为 `llmbenchlab-recovery-manifest-v1`，UTF-8、canonical { "schema": "llmbenchlab-recovery-manifest-v1", "backup_set_id": "00000000-0000-4000-8000-000000000000", - "created_at_utc": "2026-08-28T00:00:00.000000Z", - "source_alembic_head": "20260828_0005", + "created_at_utc": "2026-08-30T00:00:00.000000Z", + "source_alembic_head": "20260829_0006", "source_git_commit": null, "database_dump": { "format": "postgresql-custom", diff --git a/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md b/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md new file mode 100644 index 0000000..ac8965e --- /dev/null +++ b/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md @@ -0,0 +1,49 @@ +# ADR-0017：以 schema-equivalent 前向 revision 修复早期治理索引缺口 + +- **Status**: Accepted +- **Date**: 2026-08-30 +- **Deciders**: LLMBenchLab maintainers +- **Scope**: 早期 `20260827_0004` 本地数据库兼容修复、audit archive head 兼容与 P2-07 recovery head +- **Amends**: [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的 archive-v1 compatible-head allowlist，以及 [ADR-0016](ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) 的 P2-07 exact recovery head +- **Preserves**: canonical 数据模型、13 表/importer、audit archive v1 字段语义、`llmbenchlab-protocol-v1`、未知 drift fail-closed 与数据库外 keyring + +## Context + +一个重建前 SQLite 备份完整性与外键均正常，Alembic marker 为 `20260827_0004`，但精确缺少三个后来出现在 canonical `0004` 文件中的索引： + +- `evaluation_runs.ix_evaluation_runs_started_at_id`； +- `evaluation_runs.ix_evaluation_runs_finished_at_id`； +- `governance_policies.uq_governance_policies_single_active`。 + +因此 migration preflight 正确识别到 revision/schema 不一致，并在 `0005` 有机会运行前拒绝。删库重建会绕过而不是解决兼容问题；修改已经发布的 `0004/0005` 又会继续制造同 revision 多种含义。现有 ORM metadata、13 张应用表、archive event schema 和恢复摘要字段都不需要变化。 + +[ADR-0016](ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) 已把尚未实施的 recovery-manifest-v1 精确绑定到当时的 head `0005`，并要求未来 head 通过显式决定更新。本 ADR 是该显式 amendment。 + +## Decision + +1. 新增前向 Alembic revision `20260829_0006`，`down_revision=20260828_0005`。它不增加业务表、字段或约束，只在缺失时补齐上述三个 canonical `0004` 索引。migration 会拒绝 reflection-visible 列序、唯一性或双方言 partial predicate 不匹配的同名索引；SQLite `DESC`/`COLLATE` 等反射不完整的 DDL modifier 继续由标准入口中先执行的 deep preflight 拒绝。 +2. SQLite migration preflight 只对白名单 versioned `0004/0005` fingerprint 放行：schema 可以是 canonical，或仅缺这三个索引的任意非空子集。这样 SQLite 非事务 DDL 在 `0006` 内中断后可从 `0005` marker 重入；完整 schema、DDL modifiers、integrity、FK、已存在 repair index 的精确定义和其他索引仍严格匹配，任何额外 drift 都拒绝。 +3. 新近成为 historical 的 PostgreSQL `0005` 在 preflight 中必须重新执行 metadata diff；只接受 canonical schema 或同一 repair-index 缺失子集，不能因 revision 已知而绕过 drift 校验。SQLite preflight 与 `0006` migration 都在首条 repair DDL 前确认 active policy 不超过一条；双方言直接执行 migration 也受同一数据 guard 保护。工具绝不自动挑选或停用 policy；重复 active 数据必须由操作者核对后另行处理。 +4. `0006 -> 0005` 只回退 revision marker并保留三个索引，因为它们本来就属于 canonical `0004/0005` schema。`0005 -> 0004` 的 Worker facts guard 与 `0004 -> 0003` 的 governance/audit guard 不变。 +5. Audit archive v1 compatible-head allowlist 同时保留 `0005` 并加入 schema-equivalent `0006`；冻结的 `0005` archive fixtures 与 digest 不改写。`0006` 没有改变任何 archive event、retention 或 restore 语义。 +6. P2-07 尚未实施，因此 recovery-manifest-v1 的 exact source/target head 从 `0005` 显式修订为当前 `0006`，不建立多 head 恢复 allowlist。历史勘察中“当时 head 为 `0005`”仍作为事实保留。 + +## Consequences + +- 受影响旧库可由 `make migrate` 在一致性备份后无损前进，不再要求删库或 blind stamp；标准 `0004/0005` 数据库在 `0006` 中为 schema no-op。 +- 当前应用启动仍只接受唯一 Alembic head。升级代码后必须先由唯一 migration owner 运行 `make migrate`；API/Worker 不自行迁移。 +- 支持的 migration owner 入口始终先执行 preflight，再执行 Alembic。migration 内的 active-policy 与 reflection-visible 同名索引门禁是纵深保护，不替代 SQLite preflight 对 `DESC`/`COLLATE` 等深层 DDL modifier 的检查；裸 `alembic upgrade` 绕过 preflight 不属于受支持的旧库修复入口。 +- 这不是新的产品 schema、API、Benchmark、评分协议、数据表或安全授权模型，也不推进 P2-07 功能实现。 + +## Rejected alternatives + +- **删除并重建数据库**：会丢失或放弃旧事实，且无法修复下一个同类环境。 +- **修改既有 `0004` 或 `0005`**：继续让一个 revision 代表多个历史结构，无法修复已经位于 `0005` 的缺口数据库。 +- **在 preflight 中直接执行 repair DDL**：绕过 Alembic 的前向 schema owner，并且不能一致覆盖 PostgreSQL 或 migration 自身的纵深数据门禁。 +- **放宽通用 fingerprint**：会把未知 drift 误当兼容结构，违反 fail-closed 边界。 + +## Validation + +- migration 回归必须覆盖 `0004 gap -> 0005 -> 0006`、`0005` 中断重入、数据计数保持、partial unique enforcement、重复 active policy、同名错误索引、额外 drift、canonical no-op 与 downgrade marker 行为。 +- 真实失败备份只在临时副本上执行 prepare/upgrade/check，并核对 revision、integrity、FK、三个索引和匿名表计数；原件不修改。 +- 本地 SQLite、双方言 CI migration/check、完整测试、lint、Mock smoke、Compose config 和精确 SHA 远程 job 共同构成交付门禁。 diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 5b38a2a..53ce1b2 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -13,6 +13,7 @@ - 镜像指纹修正：[ADR-0013](../decisions/ADR-0013-stable-image-content-fingerprint.md) - 双 backlog 资格：[ADR-0014](../decisions/ADR-0014-dual-backlog-slo-profile.md) - 可观测性与审计保留：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) +- Schema-equivalent 索引修复：[ADR-0017](../decisions/ADR-0017-schema-equivalent-governance-index-repair.md) ## 阶段目标 @@ -20,7 +21,7 @@ ## 当前功能范围 -- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容、双方言 Alembic 链已在 P2-06 implementation SHA `9a20676…` 扩展至 `20260828_0005`。 +- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容；P2-06 implementation SHA `9a20676…` 将双方言 Alembic 链扩展至 `20260828_0005`，当前 schema-equivalent maintenance head `20260829_0006` 仅修复早期 `0004` 三索引缺口。 - Redis at-least-once 通知；Run、取消、重试、租约、Response、终态、治理、attempt ledger 和 audit 全由数据库裁决。 - 原子 claim、数据库时间 lease/heartbeat、fencing、有限 retry/backoff、取消、过期接管、duplicate no-op 和 dead-letter。 - 停写只读 SQLite→空 PostgreSQL 的单向 importer；`0005` 按依赖顺序复制 13 张应用表并做 count/PK/content fingerprint，源有 live Worker generation 时拒绝，stopped/stale progress 可精确复制。keyring 仍在数据库之外。 @@ -81,6 +82,7 @@ - [x] **远程实现门禁**：GitHub Actions run [`33099260233`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33099260233) 对精确实现 SHA 4/4 成功。 - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 - [x] **P2-06 仓库级闭环完成**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确 4/4 通过。 +- [ ] **0004 历史索引兼容修复待远程门禁**：schema-equivalent `20260829_0006`、仅允许三个已知索引缺失子集的可重入 SQLite preflight、PostgreSQL `0005` metadata 白名单控制流、重复 active/额外 drift 拒绝、真实失败备份副本升级及本任务已执行的本地门禁均通过；historical PG missing-index 分支仅有 Mock 回归，标准 CI 真实 PG 只覆盖 fresh canonical 分支。本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance，精确 SHA CI 绿色前保持 `local_verified`。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -110,6 +112,7 @@ | P2-06 补充 Ruff | 过宽 scripts 命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | 保留首次结果，不扩大本切片清理范围 | | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | | P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | +| 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；exact-SHA remote gate 待执行，不改变 P2-07 planned 状态 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 diff --git a/docs/plans/2026-08-28-phase-2-recovery-operations.md b/docs/plans/2026-08-28-phase-2-recovery-operations.md index 6157dc1..bf6c3f6 100644 --- a/docs/plans/2026-08-28-phase-2-recovery-operations.md +++ b/docs/plans/2026-08-28-phase-2-recovery-operations.md @@ -3,10 +3,10 @@ - Owner: Codex - Status: in_progress - Created: 2026-08-28 -- Updated: 2026-08-28 +- Updated: 2026-08-30 - Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) - Worklog: [2026-08-28 工作日志](../worklogs/2026-08-28-phase-2-recovery-operations.md) -- ADRs: [ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0007](../decisions/ADR-0007-web-provider-credentials.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md)、[ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) +- ADRs: [ADR-0005](../decisions/ADR-0005-durable-task-execution.md)、[ADR-0007](../decisions/ADR-0007-web-provider-credentials.md)、[ADR-0009](../decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md)、[ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、[ADR-0017](../decisions/ADR-0017-schema-equivalent-governance-index-repair.md) ## Context diff --git a/docs/plans/2026-08-29-database-startup-recovery.md b/docs/plans/2026-08-29-database-startup-recovery.md new file mode 100644 index 0000000..9df0f43 --- /dev/null +++ b/docs/plans/2026-08-29-database-startup-recovery.md @@ -0,0 +1,108 @@ +# 数据库启动与迁移恢复修复执行计划 + +- Owner: Codex +- Status: active +- Created: 2026-08-29 +- Updated: 2026-08-30 +- Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- Worklog: [2026-08-29 数据库启动恢复工作日志](../worklogs/2026-08-29-database-startup-recovery.md) +- ADRs: [ADR-0002 SQLite-first](../decisions/ADR-0002-sqlite-first.md)、[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md)、[ADR-0017](../decisions/ADR-0017-schema-equivalent-governance-index-repair.md)；ADR-0017 显式记录前向 repair 与 accepted P2-07 exact-head amendment + +## Context + +用户报告旧数据库无法运行，只有重建数据库后才能启动。当前配置指向本地 SQLite；现库已在 Alembic `20260828_0005`，而重建前备份记录了一个早期 `20260827_0004` 结构变体：revision 正确、完整性与外键正常，但缺少后来加入同一 revision 文件的三个索引。启动必须继续拒绝未知/损坏 schema，但这一精确历史变体应能由 `make setup` / `make migrate` 备份并无损升级，不能要求用户删库。 + +## Objective + +定位并修复受支持旧 SQLite 数据库无法升级或启动的原因，使标准 `0004/0005` 和仅缺三个已知治理索引的历史变体可在保留数据和自动备份的前提下升级到新的前向修复 revision `0006`，同时继续 fail closed 拒绝真正的 schema 漂移。 + +## Scope + +- SQLite migration preflight、Alembic `0004 -> 0005 -> 0006` 升级与本地启动入口。 +- 基于重建前备份副本的可复现诊断。 +- 针对根因的最小代码/迁移修复和回归测试。 +- 必要的 setup/deployment/troubleshooting 与状态文档同步。 + +## Non-goals + +- 不修改、删除或降级任何旧失败备份；当前重建库仅在修复和门禁通过后经标准 preflight 备份并前向升级。 +- 不迁移默认用户数据到 PostgreSQL，不改变 API、评分协议或数据模型语义。 +- 不恢复用户已主动删除的数据；只保证受支持数据库可无损迁移和启动。 +- 不推进 P2-07 recovery verifier、Redis rebuild 或告警演练。 + +## Assumptions + +- `backend/data/llmbenchlab.db.pre-alembic-20260828T121151372702Z.bak` 是失败前由迁移流程生成的只读一致性备份；只在临时副本上执行迁移。 +- 当前 `.env` 的相对 SQLite URL由仓库脚本在 `backend/` 目录解析，目标是 `backend/data/llmbenchlab.db`。 +- 根因已由 `repair-before-0005` 备份的 schema fingerprint 精确复现；另一份 `pre-alembic` 备份已含三个索引并可正常升级。 + +## Requirements + +- 保留 ADR-0002 的 Alembic-only schema owner 与无损迁移原则。 +- 不把 `create_all`、删库、stamp 猜测或降低校验作为修复。 +- 测试必须先复现旧行为，再证明数据、revision、约束和索引都正确。 +- 自动化只用临时 SQLite/Mock，不调用真实 Provider。 + +## Implementation steps + +1. [completed] 在迁移前备份的临时副本上复现并锁定根因。 + - Files/modules: `scripts/migrate.sh`、`backend/app/db/prepare_migrations.py`、`backend/alembic/versions/20260828_0005_worker_progress_retention.py`。 + - Validation: 捕获稳定错误、失败前后 revision/schema/data 摘要，不写原始备份。 +2. [completed] 新增失败回归并实施最小修复。 + - Files/modules: migration/preflight 与 `backend/tests/test_migrations.py`，只按根因扩展。 + - Validation: 旧回归在修复前失败、修复后通过；未知/partial schema 仍拒绝。 +3. [completed] 验证新库、旧 head、重复迁移和真实启动路径。 + - Validation: 目标 pytest、Alembic upgrade/check、临时副本数据保持、API/Worker startup probe、Ruff。 +4. [in_progress] 同步运维/状态文档并完成交付门禁。 + - Validation: diff/secret/staged review、commit/push、精确 SHA GitHub Actions 四 job 全绿。 + +## Risks + +| 风险 | 可能性/影响 | 预防措施 | 触发后的处理 | +|---|---|---|---| +| 误改用户当前库 | 低/高 | 失败现场只用临时副本；当前重建库确认无服务写入后只走自动备份的标准前向 migration | 立即停止，保留自动备份和现场，不做额外写入 | +| 把 schema 漂移误当可恢复 | 中/高 | 只接受精确已知 revision/fingerprint；保留 fail-closed 测试 | 拒绝自动修复并报告具体稳定错误 | +| SQLite 非事务 DDL 留下半迁移 | 中/高 | 迁移前一致性备份、测试失败重入与结构探测 | 从备份副本复核；不得 stamp 猜测 | +| 相对 URL 指向不同文件 | 中/中 | 测试 root/backend 两种 cwd，统一解析/诊断 | 固定入口解析并更新文档 | + +## Validation + +| 验收项 | 命令或检查 | 预期结果 | 实际结果 | +|---|---|---|---| +| 旧库复现 | 两个 `0004` 备份只读 fingerprint + 临时副本 preflight | 缺三索引变体稳定触发既有严格拒绝；标准变体正常 | 已确认；原件未修改 | +| Migration tests | `cd backend && uv run pytest tests/test_migrations.py` | 全部通过 | `52 passed` | +| SQLite round trip | 真实失败备份副本 `0004 -> 0005 -> 0006 -> check` | 数据/约束/index/revision 正确 | 通过；integrity ok、FK 0、三索引存在、匿名计数保持 | +| Startup | 当前重建库标准 migrate + head gate | 能启动并只连预期库 | 通过；备份后到 `0006`，`initialize_database()`=0 | +| Full gates | `make lint`、`make test`、`make smoke`、frontend build、Compose config | 全部通过 | 通过；后端 927/33、前端 38、smoke 1/7、build/config exit 0 | +| Docs/security | links、diff、secret、staged review | 无未解决 Blocker/High/Medium | diff check 已通过；staged 终审待 commit 前执行 | +| Remote | 普通 push + exact-SHA CI | 四个 required job 全绿 | 待执行 | + +## Rollback + +代码修复可回退；迁移回归和失败现场验证仅作用于临时副本。当前重建库已通过标准 preflight 生成一致性备份后从 canonical `0005` 前进到 schema-equivalent `0006`；downgrade 到 `0005` 只回退 marker，但正常使用仍应保留当前 head。现有旧 `.bak` 不删除或改写。若新逻辑不能证明已知 schema exact match，则保持拒绝并让操作者从原备份恢复，而不是自动 stamp、drop 或重建。 + +## Documentation updates + +- [x] README / setup 与旧库升级说明 +- [x] Deployment / migration 排障与恢复路径 +- [x] Testing / 新回归与实际结果 +- [x] CHANGELOG、PROJECT_STATUS、Phase 2、NEXT_TASK、工作日志 +- [x] API / Benchmark：不适用；Security 已同步 fail-closed compatibility 边界 + +## Completion evidence + +- Changed files: migration/preflight/audit compatibility/acceptance constants、回归、README 与运维/安全/测试/状态文档。 +- Commands run: migration 52 tests、真实失败备份副本 upgrade/check、`make lint/test/smoke`、Compose config、当前库 migrate/startup/check。 +- Acceptance evidence: 本任务已执行的本地门禁全部通过；远程 exact-SHA 待 push 后记录。 +- Not run: 本地真实 PostgreSQL integration 与完整 Compose acceptance 未单独重跑；required CI 的 PostgreSQL/integration/full-stack job 将作为精确 SHA 远程门禁。historical PostgreSQL `0005` 缺索引的 preflight/create 分支只有 Mock 控制流回归；标准 CI 的真实 PostgreSQL migration 覆盖 fresh canonical existing-index 分支，不把两者混写。 +- Known issues: root `data/llmbenchlab.db` 仍是未由统一 Make 入口使用的旧 `0003` 文件；未删除。旧失败备份保留，是否含需恢复的业务数据由用户决定。 + +## Decision and discovery log + +| 日期 | 类型 | 记录 | 影响/后续 | +|---|---|---|---| +| 2026-08-29 | discovery | 当前库为 SQLite `20260828_0005`，quick check 正常；仓库另有 root 下的旧 `0003` 文件，但 Make 入口从 `backend/` 解析数据库。 | 不修改当前库；从配置入口和重建前备份复现。 | +| 2026-08-29 | discovery | 重建前两个 100 MB 备份均 quick-check 正常、revision=`20260827_0004`、无 `worker_processes`/0005 indexes。 | 这是精确的旧 head 输入，可在临时副本验证无损升级。 | +| 2026-08-29 | root cause | `repair-before-0005` 比 canonical `0004` 精确缺少 Run started/finished 游标索引和 single-active policy 唯一部分索引；`pre-alembic` 备份已补齐三者。 | 只对白名单 fingerprint 放行，由新 `0006` 条件补建；未知 drift 与多 active policy 继续拒绝。 | +| 2026-08-29 | local delivery | 修复、全量门禁和失败备份副本验证通过后，当前重建库由 `make migrate` 自动备份并前进到 `0006`；startup/check 通过，数据计数保持。 | 当前本地运行阻塞解除；进入 staged/remote gate。 | +| 2026-08-30 | decision | 新 head 影响 ADR-0015 archive allowlist 与已接受 ADR-0016 的 P2-07 exact recovery head。 | 新增 accepted ADR-0017 显式 amendment，并同步 P2-07 plan/worklog/NEXT；不把这项合同变化藏在实现或旧 ADR 中。 | diff --git a/docs/worklogs/2026-08-28-phase-2-recovery-operations.md b/docs/worklogs/2026-08-28-phase-2-recovery-operations.md index b9cec9a..a2b3335 100644 --- a/docs/worklogs/2026-08-28-phase-2-recovery-operations.md +++ b/docs/worklogs/2026-08-28-phase-2-recovery-operations.md @@ -39,7 +39,7 @@ P2-06 已在 implementation SHA `9a20676dcf545040782f04c166205d0043345753`、evi ## 验收标准 -- 正确 dump+keyring 在新空 PG16 目标恢复后 head=`20260828_0005`、13 表 count/PK/content digest 与治理/audit 完整性全 exact。 +- 按后续 [ADR-0017](../decisions/ADR-0017-schema-equivalent-governance-index-repair.md) amendment，正确 dump+keyring 在新空 PG16 目标恢复后 head=`20260829_0006`、13 表 count/PK/content digest 与治理/audit 完整性全 exact。 - Manifest/dump/keyring swap、missing/wrong/权限/类型/size/race/tamper 全部 fail closed；stored fake credential 全部本地解密且 Provider I/O=0。 - Redis container/volume 完全替换后，DB-only scan 恢复 pending/expired、group 从 `0-0` 重建、PEL/lag=0、重复通知不改变 durable snapshot。 - Worker 1↔2 扩缩的 expected/live/stalled/shortfall、graceful stop、lease/fencing/ledger/Response 唯一性收敛。 diff --git a/docs/worklogs/2026-08-29-database-startup-recovery.md b/docs/worklogs/2026-08-29-database-startup-recovery.md new file mode 100644 index 0000000..9ff43c9 --- /dev/null +++ b/docs/worklogs/2026-08-29-database-startup-recovery.md @@ -0,0 +1,90 @@ +# 2026-08-29 — 数据库启动与迁移恢复工作日志 + +## 元信息 + +- 日期：2026-08-29 +- 执行者：Codex +- 分支：`codex/complete-evaluation-workflow` +- 初始 HEAD：`6df84d9bb26ee81c2f91ed9772dd6f45e9e17741` +- 计划：[数据库启动与迁移恢复修复](../plans/2026-08-29-database-startup-recovery.md) +- 阶段：[Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- 状态：`in_progress` + +## 目标与背景 + +用户反馈旧数据库无法运行，只有重建数据库后才能启动。本任务先利用仓库保留的迁移前一致性备份安全复现，再修复受支持旧库的迁移/启动路径；不能把“删库重建”当解决方案。 + +## 范围 + +- Alembic head gate、SQLite migration preflight 和 `0004 -> 0005 -> 0006` 前向修复。 +- 临时副本复现、回归测试、最小修复与必要文档。 +- 本地/远程交付门禁。 + +## 非目标 + +- 诊断与回归不写原始 `.bak`；当前重建库只在生产修复和全量门禁通过后，经标准 preflight 自动备份再执行一次前向 migration。 +- 不恢复已删除业务内容，不推进 P2-07，不改变 API/协议/依赖。 +- 不通过 `create_all`、盲 stamp、drop 或降低完整性检查来过绿。 + +## 验收标准 + +- 重建前缺三个索引的 `0004` 备份临时副本能无损升级到 `0006` 并通过 `alembic check`。 +- migration 可重复，当前 head 新库仍正常；partial/unknown schema 仍 fail closed。 +- 启动入口连接预期数据库，API/Worker 不再要求重建库。 +- 目标/全量测试、lint、smoke、文档/秘密/staged review 与 exact-SHA CI 通过。 + +## 假设与风险 + +- 备份是失败前一致性快照，只读原件、仅复制到临时目录。 +- SQLite DDL 非事务性，故必须验证失败重入与部分结构，不能只测空库。 +- 仓库存在 root `data/llmbenchlab.db` 与 backend `backend/data/llmbenchlab.db` 两个文件；必须确认真实入口，避免修错文件。 + +## 实施步骤 + +1. 只读核对配置、当前库、旧备份、Alembic revision 和启动入口。 +2. 在临时副本复现失败并记录稳定根因。 +3. 先补回归，再实施最小迁移/启动修复。 +4. 完成目标/全量/启动验证和文档同步。 +5. staged 审查、独立 commit、普通 push 与精确 SHA CI。 + +## 初始勘察 + +- 初始工作树 clean，HEAD 与 origin 同步。 +- 当前 backend SQLite quick check 正常，revision=`20260828_0005`；业务表为空，存在两条 Worker generation 行。 +- root 下另有 revision=`20260827_0003` 的小型旧 SQLite，但 Make 启动入口进入 `backend/` 后再加载相对 URL。 +- 两个重建前 100 MB 备份 quick check 正常，revision=`20260827_0004`，且都没有 0005 新表/索引。`repair-before-0005` 精确缺少三个 canonical 0004 索引；随后的 `pre-alembic` 备份已补齐三者。 +- 当前没有运行中的 LLMBenchLab 进程或 Compose service。 +- 根因不是 SQLite 损坏：旧库在早期 0004 文件执行后已写入 0004 revision，随后同一 migration 文件增加了 `ix_evaluation_runs_started_at_id`、`ix_evaluation_runs_finished_at_id` 和 `uq_governance_policies_single_active`。当前 preflight 看到 revision/schema 不一致后按设计拒绝，在 0005 有机会运行前退出。 + +## 实际修改 + +- 新增 `20260829_0006` 前向 repair revision：先拒绝多条 active policy，再条件补齐三个 canonical governance indexes；已有同名索引的 reflection-visible 列序、唯一性和 partial predicate 必须匹配，SQLite 反射不完整的 DDL modifier 由标准 preflight 深检。downgrade 只回退 revision marker，保留本属于 canonical `0004` 的索引。 +- `prepare_migrations` 将 `0005` 纳入 historical validation，只对白名单 canonical `0004/0005` 或三个 repair 索引的缺失子集放行，并在 SQLite 一致性备份前完成完整 schema/integrity/FK、索引和 single-active 行校验；新近成为 historical 的 PostgreSQL `0005` 也执行 metadata diff，任何额外 drift 继续拒绝。 +- migration 回归覆盖真实历史形态、`0005` 部分 DDL 后中断重入、PostgreSQL historical drift、数据保持、错误索引定义、重复 active policy、额外 drift 和 no-op downgrade。 +- audit archive v1 明确兼容 schema-equivalent `0005/0006`；Compose acceptance 分离 current head `0006` 与 Worker progress boundary `0005`。 +- 新增并接受 ADR-0017，显式记录 repair revision、archive-v1 双 head 兼容，以及对已接受 ADR-0016/P2-07 exact recovery head 的 `0006` amendment；没有静默改写既有决定。 +- 同步 README、Architecture、Security、Deployment、Operations、Testing、ADR-0015/0016、Phase 2、NEXT_TASK、CHANGELOG、状态/计划/日志。 +- 在实现与全量门禁通过后，对当前 `backend/data/llmbenchlab.db` 执行标准 `make migrate`；preflight 生成 `llmbenchlab.db.pre-alembic-20260829T155710705255Z.bak`，随后仅推进 `0005 -> 0006`。 + +## 已运行命令与结果 + +- 强制文档、ADR-0002、Deployment/Testing、Make/setup/dev/migrate、migration/preflight 源码只读检查：完成。 +- `git status --short --branch`：初始 clean。 +- 两个数据库及备份的 SQLite `quick_check`/revision/table/index 匿名摘要：完成，未修改原文件。 +- `repair-before-0005` 真实失败备份临时副本：`prepare`、`0004 -> 0005 -> 0006`、`alembic check` 全部通过；最终 integrity=`ok`、FK violations=`0`、三个 repair index 全部存在、匿名业务表计数保持；原件未修改，临时副本已清理。 +- `cd backend && uv run pytest tests/test_migrations.py`：`52 passed`（含 SQLite partial-DDL resume 与 PostgreSQL historical `0005` drift/行门禁控制流）。 +- 迁移/audit/archive/acceptance-script 定向集合：通过；包含新增 archive-v1 `0006` compatibility 回归。 +- `make lint`：通过；Ruff check、153 files format check、ESLint、TypeScript typecheck 均为 exit 0。 +- 最终 `make test`：通过；后端 `927 passed, 33 skipped, 1238 warnings`，前端 `38 passed`；warning 为既有 Starlette/httpx、Python 3.14 pytest-asyncio 弃用提示及迁移负向用例的 SQLAlchemy/Alembic warning。 +- `npm --prefix frontend run build`：通过；2192 modules，保留既有 662.39 kB chunk warning。 +- `make smoke`：`1 passed, 7 deselected`，完全离线 Mock。 +- `docker compose config --quiet`：exit 0；未启动长期服务。 +- 当前重建库 `make migrate`：exit 0，自动备份后 revision=`20260829_0006`；随后 `initialize_database()`=0、quick check=`ok`、FK violations=`0`、repair indexes=`3`、`alembic check` 为 `No new upgrade operations detected`。迁移前后业务表均为 0 行，既有 `worker_processes=2` 保持。 + +## 已知问题与下一步 + +- 当前新库已在自动备份后完成 `0005 -> 0006`，完整性、外键、schema fingerprint、startup head gate 与 `alembic check` 均正常；业务表计数和 Worker facts 保持。 +- 修复只接受 canonical schema 或三个已知索引的缺失子集，并在补建唯一部分索引前拒绝多条 active policy；不会自动挑选或改写治理事实，额外 drift 仍拒绝。 +- root `data/llmbenchlab.db` 是旧 `0003` 文件；支持的 `make setup/migrate/dev/backend/worker` 入口都会进入 `backend/`，本任务不删除或合并这个旧文件。不要从仓库 root 直接以相对 SQLite URL 绕过统一入口。 +- 本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance；required CI 的 PostgreSQL/integration/full-stack job 将作为精确 SHA 远程门禁，不把历史 P2-06 evidence 冒充本次运行。historical PostgreSQL `0005` 缺索引的 preflight/create 分支本次只有 Mock 控制流回归；标准真实 PostgreSQL CI 覆盖 fresh canonical existing-index 分支，不宣称覆盖该历史缺口。 +- 本地实现、当前库和回归已验证；尚待 staged review、普通 push 与精确 SHA GitHub Actions 必需 job。绿色前状态保持 `in_progress`。 diff --git a/scripts/phase2_acceptance.py b/scripts/phase2_acceptance.py index b231c8b..be3340f 100644 --- a/scripts/phase2_acceptance.py +++ b/scripts/phase2_acceptance.py @@ -43,6 +43,7 @@ PRE_GOVERNANCE_REVISION = "20260827_0003" GOVERNANCE_REVISION = "20260827_0004" WORKER_PROGRESS_REVISION = "20260828_0005" +DATABASE_HEAD_REVISION = "20260829_0006" TASK_MESSAGE_VERSION = "llmbenchlab-run-task-v1" TASK_STREAM = "llmbenchlab:runs:v1" TASK_GROUP = "llmbenchlab-workers-v1" @@ -2367,7 +2368,7 @@ def migration_round_trip_scenario(self) -> None: "SELECT version_num FROM alembic_version;" ).stdout.strip() self.require( - application_revision == WORKER_PROGRESS_REVISION, + application_revision == DATABASE_HEAD_REVISION, "failed populated downgrade changed the application database revision", application_revision, ) @@ -2545,8 +2546,8 @@ def migration_round_trip_scenario(self) -> None: database=empty_database, ).stdout.strip() self.require( - empty_head_before == WORKER_PROGRESS_REVISION, - "empty PostgreSQL database did not upgrade to 0005", + empty_head_before == DATABASE_HEAD_REVISION, + "empty PostgreSQL database did not upgrade to current head", empty_head_before, ) downgrade_empty = self.compose( @@ -2672,7 +2673,7 @@ def migration_round_trip_scenario(self) -> None: database=empty_database, ).stdout.strip() self.require( - empty_final_head == WORKER_PROGRESS_REVISION, + empty_final_head == DATABASE_HEAD_REVISION, "empty PostgreSQL database did not return to migration head", empty_final_head, ) From 3b1471a9891fe9a53676a52badc43fa14b5c25bd Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 00:42:18 +0800 Subject: [PATCH 06/12] docs: close database compatibility repair --- docs/NEXT_TASK.md | 2 +- docs/PROJECT_STATUS.md | 9 +++++---- docs/TESTING.md | 2 +- docs/phases/PHASE-2-RELIABILITY.md | 4 ++-- docs/plans/2026-08-29-database-startup-recovery.md | 12 ++++++------ .../worklogs/2026-08-29-database-startup-recovery.md | 8 +++++--- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 3a83317..4f29b41 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -20,7 +20,7 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 提供 canonical JSONL v1、严格文件/权限/大小/schema/hash/rollup 校验、真正离线 verify、精确 digest 绑定、默认不删除和 commit-outcome 分类。它不提供签名/WORM，也不替代整库+keyring backup。 - 全生产日志调用受字面量/无格式参数静态门禁；第三方动态消息固定化、结构化数值有限化，raw Uvicorn access handler 关闭。 -`0006` 兼容修复已完成本地 migration/full test/lint/Mock smoke、真实失败备份副本升级和当前重建库 startup/check；精确 SHA 远程 CI 尚待执行，因此当前标记为 `local_verified`，不改变 P2-07 的 `planned` 状态或范围。 +`0006` 兼容修复已完成本地 migration/full test/lint/Mock smoke、真实失败备份副本升级和当前重建库 startup/check；实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 4/4 成功，因此该维护任务为 `completed`，不改变 P2-07 的 `planned` 状态或范围。 P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index f3a5fb5..8bc3f60 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。当前 [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / `20260829_0006` 兼容修复仅为 `local_verified`，仍位于未提交工作树，尚未 push 且没有精确 SHA 远程 CI；上述 P2-06 历史绿灯不替代本次门禁。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。[ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / `20260829_0006` 数据库兼容修复实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已普通 push，其精确 SHA 的 GitHub Actions [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 四个必需 job 全部成功，因此该维护任务为 `completed`。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -50,16 +50,17 @@ - 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 - 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。Evidence closeout 文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push，精确 SHA run `33165775037` 4/4 成功，P2-06 仓库级收尾完成。P2-06 当时默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 -## 2026-08-29 数据库兼容修复（`local_verified`） +## 2026-08-29 数据库兼容修复（`completed`） - 旧库失败不是 SQLite 损坏，而是 revision=`20260827_0004` 的早期结构变体精确缺少三个后来加入 canonical `0004` 的索引；严格 preflight 因 revision/schema 不一致，在 `0005` 执行前按设计拒绝。 - [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) 已接受；当前 head 为 schema-equivalent `20260829_0006`。preflight 只接受 canonical `0004/0005`，或仅缺一至三个已知 repair 索引的 fingerprint，以支持 SQLite repair 中断重入；PostgreSQL `0005` 的额外 metadata drift 同样拒绝。完整 schema/integrity/FK、索引定义、single-active policy 数据门禁仍 fail closed。`0006` 首条 DDL 前再次拒绝多条 active policy，并条件补建索引；`0006 -> 0005` 保留 canonical 对象。 - 真实失败备份副本已无损升级到 `0006` 并通过 integrity/FK/Alembic check；当前重建库由标准 preflight 自动备份后从 canonical `0005` 到 `0006`，startup gate、quick check、FK 和 metadata check 全部通过，业务计数与 Worker facts 保持。 -- 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。精确 SHA 远程 CI 尚未执行，因此本维护任务暂不标记 completed，Phase 2/P2-07 状态不变。 +- 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 的 backend、真实 PostgreSQL/Redis、完整 Compose 与 frontend 四个 job 全部成功；本维护任务完成，不改变 Phase 2/P2-07 状态。 ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 +- 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 - P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 @@ -86,7 +87,7 @@ | P2-06 Mock smoke | `1 passed, 7 deselected` | 完全离线通过 | | P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；实现 SHA 的远程 integration job 也已成功 | | P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；当时默认用户 SQLite 非 head 的 check 失败后未迁移 | P2-06 当时 head `20260828_0005`；历史证据保持原状 | -| 2026-08-29 DB compatibility repair | migration `52 passed`；最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；本地已验证，exact-SHA CI 待执行 | +| 2026-08-29 DB compatibility repair | migration `52 passed`；最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；`8fb51b6…` 的 run `33263405214` 4/4 成功 | | P2-06 build/config | frontend build 成功（保留 662.39 kB chunk warning）；Compose config exit 0 | 从根目录误跑 npm 的失败已记录并用正确目录重跑通过 | | P2-06 Prometheus 规则 | `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | 临时容器验证；仓库仍不部署 Prometheus/Alertmanager | | P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`、`0005`/isolated `0004` populated refusal、两层空库往返、cleanup C/V/N empty | diff --git a/docs/TESTING.md b/docs/TESTING.md index 7827437..895db9e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -398,7 +398,7 @@ P2-06 实现 SHA 与提交前阶段性证据如下。Dirty 结果只作为历史 | `make smoke` | `1 passed, 7 deselected`；完全离线 Mock | | Frontend production build | 从根目录误运行 `npm run build` 因无 `package.json` 失败；改为 `cd frontend && npm run build` 后成功（2192 modules），保留 662.39 kB 主 chunk 非阻断 warning | | Migration | 默认用户 SQLite 尚未在 head，直接 `alembic check` 失败且未擅自迁移；临时 SQLite head→`0001`→head/check 与隔离真实 PostgreSQL 往返/check 全绿；P2-06 当时 head `20260828_0005` | -| 2026-08-29 DB compatibility repair（本地） | 真实失败备份副本 `0004 -> 0005 -> 0006`、migration `52 passed`、最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`、当前重建库 migrate/startup/check、lint/smoke/build/config 全绿；current head `20260829_0006`。本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance；historical PG `0005` missing-index 分支仅有 Mock 控制流回归，exact-SHA CI 的真实 PG job 覆盖 fresh canonical 分支 | +| 2026-08-29 DB compatibility repair | 真实失败备份副本 `0004 -> 0005 -> 0006`、migration `52 passed`、最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`、当前重建库 migrate/startup/check、lint/smoke/build/config 全绿；实现 SHA `8fb51b6…` 的 run `33263405214` 4/4 成功。historical PG `0005` missing-index 分支仅有 Mock 控制流回归，exact-SHA CI 的真实 PG job 覆盖 fresh canonical 分支 | | Compose config | `docker compose config --quiet` exit 0 | | Prometheus rules | 临时 `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | | Ruff scripts 补充检查 | 对 `scripts/` 的过宽默认 Ruff 命令报告 93 条既有 modernization 告警；按本任务合同使用 `--select E,F,I` 后通过，未把既有告警冒充本次失败 | diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 53ce1b2..f155ed4 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -82,7 +82,7 @@ - [x] **远程实现门禁**：GitHub Actions run [`33099260233`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33099260233) 对精确实现 SHA 4/4 成功。 - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 - [x] **P2-06 仓库级闭环完成**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确 4/4 通过。 -- [ ] **0004 历史索引兼容修复待远程门禁**：schema-equivalent `20260829_0006`、仅允许三个已知索引缺失子集的可重入 SQLite preflight、PostgreSQL `0005` metadata 白名单控制流、重复 active/额外 drift 拒绝、真实失败备份副本升级及本任务已执行的本地门禁均通过；historical PG missing-index 分支仅有 Mock 回归，标准 CI 真实 PG 只覆盖 fresh canonical 分支。本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance，精确 SHA CI 绿色前保持 `local_verified`。 +- [x] **0004 历史索引兼容修复完成**：schema-equivalent `20260829_0006`、仅允许三个已知索引缺失子集的可重入 SQLite preflight、PostgreSQL `0005` metadata 白名单控制流、重复 active/额外 drift 拒绝、真实失败备份副本升级及本地门禁均通过；实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 的 [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 4/4 成功。historical PG missing-index 分支仍仅有 Mock 回归，标准 CI 真实 PG 只覆盖 fresh canonical 分支。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -112,7 +112,7 @@ | P2-06 补充 Ruff | 过宽 scripts 命令报告 93 条既有 modernization 告警；`--select E,F,I` 通过 | 保留首次结果，不扩大本切片清理范围 | | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | | P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | -| 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；exact-SHA remote gate 待执行，不改变 P2-07 planned 状态 | +| 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | `8fb51b6…` 的 exact-SHA run `33263405214` 4/4；不改变 P2-07 planned 状态 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 diff --git a/docs/plans/2026-08-29-database-startup-recovery.md b/docs/plans/2026-08-29-database-startup-recovery.md index 9df0f43..6be4c20 100644 --- a/docs/plans/2026-08-29-database-startup-recovery.md +++ b/docs/plans/2026-08-29-database-startup-recovery.md @@ -1,7 +1,7 @@ # 数据库启动与迁移恢复修复执行计划 - Owner: Codex -- Status: active +- Status: completed - Created: 2026-08-29 - Updated: 2026-08-30 - Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) @@ -53,7 +53,7 @@ - Validation: 旧回归在修复前失败、修复后通过；未知/partial schema 仍拒绝。 3. [completed] 验证新库、旧 head、重复迁移和真实启动路径。 - Validation: 目标 pytest、Alembic upgrade/check、临时副本数据保持、API/Worker startup probe、Ruff。 -4. [in_progress] 同步运维/状态文档并完成交付门禁。 +4. [completed] 同步运维/状态文档并完成交付门禁。 - Validation: diff/secret/staged review、commit/push、精确 SHA GitHub Actions 四 job 全绿。 ## Risks @@ -74,8 +74,8 @@ | SQLite round trip | 真实失败备份副本 `0004 -> 0005 -> 0006 -> check` | 数据/约束/index/revision 正确 | 通过；integrity ok、FK 0、三索引存在、匿名计数保持 | | Startup | 当前重建库标准 migrate + head gate | 能启动并只连预期库 | 通过；备份后到 `0006`，`initialize_database()`=0 | | Full gates | `make lint`、`make test`、`make smoke`、frontend build、Compose config | 全部通过 | 通过；后端 927/33、前端 38、smoke 1/7、build/config exit 0 | -| Docs/security | links、diff、secret、staged review | 无未解决 Blocker/High/Medium | diff check 已通过；staged 终审待 commit 前执行 | -| Remote | 普通 push + exact-SHA CI | 四个 required job 全绿 | 待执行 | +| Docs/security | links、diff、secret、staged review | 无未解决 Blocker/High/Medium | 通过；staged secret scan 无命中，独立终审无 Blocker/High/Medium | +| Remote | 普通 push + exact-SHA CI | 四个 required job 全绿 | 实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push；run `33263405214` 4/4 成功 | ## Rollback @@ -93,8 +93,8 @@ - Changed files: migration/preflight/audit compatibility/acceptance constants、回归、README 与运维/安全/测试/状态文档。 - Commands run: migration 52 tests、真实失败备份副本 upgrade/check、`make lint/test/smoke`、Compose config、当前库 migrate/startup/check。 -- Acceptance evidence: 本任务已执行的本地门禁全部通过；远程 exact-SHA 待 push 后记录。 -- Not run: 本地真实 PostgreSQL integration 与完整 Compose acceptance 未单独重跑；required CI 的 PostgreSQL/integration/full-stack job 将作为精确 SHA 远程门禁。historical PostgreSQL `0005` 缺索引的 preflight/create 分支只有 Mock 控制流回归；标准 CI 的真实 PostgreSQL migration 覆盖 fresh canonical existing-index 分支，不把两者混写。 +- Acceptance evidence: 本任务已执行的本地门禁全部通过；实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已普通 push，精确 SHA run `33263405214` 的 backend、PostgreSQL/Redis integration、完整 Compose 与 frontend 四个 job 全部成功。 +- Not run locally: 本地真实 PostgreSQL integration 与完整 Compose acceptance 未单独重跑；实现精确 SHA run `33263405214` 已由 PostgreSQL/integration/full-stack job 完成远程覆盖。historical PostgreSQL `0005` 缺索引的 preflight/create 分支只有 Mock 控制流回归；标准 CI 的真实 PostgreSQL migration 覆盖 fresh canonical existing-index 分支，不把两者混写。 - Known issues: root `data/llmbenchlab.db` 仍是未由统一 Make 入口使用的旧 `0003` 文件；未删除。旧失败备份保留，是否含需恢复的业务数据由用户决定。 ## Decision and discovery log diff --git a/docs/worklogs/2026-08-29-database-startup-recovery.md b/docs/worklogs/2026-08-29-database-startup-recovery.md index 9ff43c9..4ac8bda 100644 --- a/docs/worklogs/2026-08-29-database-startup-recovery.md +++ b/docs/worklogs/2026-08-29-database-startup-recovery.md @@ -8,7 +8,7 @@ - 初始 HEAD：`6df84d9bb26ee81c2f91ed9772dd6f45e9e17741` - 计划：[数据库启动与迁移恢复修复](../plans/2026-08-29-database-startup-recovery.md) - 阶段：[Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) -- 状态：`in_progress` +- 状态：`completed` ## 目标与背景 @@ -80,11 +80,13 @@ - `make smoke`：`1 passed, 7 deselected`，完全离线 Mock。 - `docker compose config --quiet`：exit 0；未启动长期服务。 - 当前重建库 `make migrate`：exit 0，自动备份后 revision=`20260829_0006`；随后 `initialize_database()`=0、quick check=`ok`、FK violations=`0`、repair indexes=`3`、`alembic check` 为 `No new upgrade operations detected`。迁移前后业务表均为 0 行，既有 `worker_processes=2` 保持。 +- staged review：`git diff --cached --check` 与秘密扫描通过，独立终审无 Blocker/High/Medium；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已普通 push 到 `origin/codex/complete-evaluation-workflow`。 +- 精确 SHA GitHub Actions [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214)：4/4 成功；backend、真实 PostgreSQL/Redis integration、完整 Compose reliability acceptance 与 frontend job 全绿。 ## 已知问题与下一步 - 当前新库已在自动备份后完成 `0005 -> 0006`，完整性、外键、schema fingerprint、startup head gate 与 `alembic check` 均正常；业务表计数和 Worker facts 保持。 - 修复只接受 canonical schema 或三个已知索引的缺失子集，并在补建唯一部分索引前拒绝多条 active policy；不会自动挑选或改写治理事实，额外 drift 仍拒绝。 - root `data/llmbenchlab.db` 是旧 `0003` 文件；支持的 `make setup/migrate/dev/backend/worker` 入口都会进入 `backend/`，本任务不删除或合并这个旧文件。不要从仓库 root 直接以相对 SQLite URL 绕过统一入口。 -- 本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance；required CI 的 PostgreSQL/integration/full-stack job 将作为精确 SHA 远程门禁，不把历史 P2-06 evidence 冒充本次运行。historical PostgreSQL `0005` 缺索引的 preflight/create 分支本次只有 Mock 控制流回归；标准真实 PostgreSQL CI 覆盖 fresh canonical existing-index 分支，不宣称覆盖该历史缺口。 -- 本地实现、当前库和回归已验证；尚待 staged review、普通 push 与精确 SHA GitHub Actions 必需 job。绿色前状态保持 `in_progress`。 +- 本地未单独重跑真实 PostgreSQL integration 或完整 Compose acceptance；实现精确 SHA run `33263405214` 的 PostgreSQL/integration/full-stack job 已完成远程覆盖，不把历史 P2-06 evidence 冒充本次运行。historical PostgreSQL `0005` 缺索引的 preflight/create 分支本次只有 Mock 控制流回归；标准真实 PostgreSQL CI 覆盖 fresh canonical existing-index 分支，不宣称覆盖该历史缺口。 +- 实现、当前库、回归、普通 push 与实现精确 SHA 远程门禁均已完成；本任务停止于数据库兼容修复，不推进 P2-07。 From 5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 01:15:47 +0800 Subject: [PATCH 07/12] fix: restore local data and quiet dev startup --- CHANGELOG.md | 1 + README.md | 4 +- backend/tests/test_dev_script.py | 185 ++++++++++++++++++ docs/DEPLOYMENT.md | 13 +- docs/NEXT_TASK.md | 2 + docs/OPERATIONS.md | 4 + docs/PROJECT_STATUS.md | 12 +- docs/SECURITY.md | 1 + docs/TESTING.md | 4 +- docs/phases/PHASE-2-RELIABILITY.md | 3 + .../2026-08-30-restore-data-quiet-startup.md | 106 ++++++++++ .../2026-08-30-restore-data-quiet-startup.md | 159 +++++++++++++++ scripts/dev.sh | 58 ++++-- 13 files changed, 531 insertions(+), 21 deletions(-) create mode 100644 backend/tests/test_dev_script.py create mode 100644 docs/plans/2026-08-30-restore-data-quiet-startup.md create mode 100644 docs/worklogs/2026-08-30-restore-data-quiet-startup.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 593431f..802001e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Changed +- Changed the combined `make dev` launcher to keep the console quiet after a concise address/log summary. API, Worker, and Vite output is appended to separate Git-ignored `artifacts/dev-logs` files with UTC session markers and private local permissions; individual service Make targets remain foreground diagnostics, and child failures still propagate their status. - Governed production logging at the source boundary: application calls use literal argument-free messages, structured values remain allowlisted and finite, third-party logger text/identity is replaced by fixed classification, and the raw Uvicorn access handler is disabled rather than bypassing the JSON redaction contract. - Extended the Phase 2 migration/importer head from `20260827_0004` / 12 tables to `20260828_0005` / 13 tables. A populated `worker_processes` table blocks `0005 -> 0004` before DDL; only an explicitly emptied table may downgrade, after which the existing governance/audit `0004` guards still apply. - Hardened the final P2-06 integrity boundaries: importer reconciliation now compares committed target facts through the same canonical contract, no-op retention mutations still execute post-commit verification, PostgreSQL archive/delete/restore operations hold advisory plus row locks, and audit archives enforce the record-count cap before per-line decoding. diff --git a/README.md b/README.md index 4fa7c03..8480ea6 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ make setup make dev ``` -`make setup` 会让 `uv` 显式选择 CPython，按锁文件安装前后端依赖、仅在 `.env` 不存在时从 `.env.example` 创建它，并执行 Alembic migration；已有 `.env` 不会被覆盖。该命令可重复执行。若检测到由早期开发版自动建表留下的未版本化 SQLite，只有在结构与完整性严格匹配已知版本时才会先创建同目录 `.bak` 一致性备份并无损收养；未知或部分结构会在写入版本标记前停止。普通 API/Worker 启动不会隐式建表，未迁移时会提示先运行 `make setup` 或 `make migrate`。`make dev` 在一个终端启动 API、独立 Worker 和 frontend，`Ctrl-C` 会一起停止。 +`make setup` 会让 `uv` 显式选择 CPython，按锁文件安装前后端依赖、仅在 `.env` 不存在时从 `.env.example` 创建它，并执行 Alembic migration；已有 `.env` 不会被覆盖。该命令可重复执行。若检测到由早期开发版自动建表留下的未版本化 SQLite，只有在结构与完整性严格匹配已知版本时才会先创建同目录 `.bak` 一致性备份并无损收养；未知或部分结构会在写入版本标记前停止。普通 API/Worker 启动不会隐式建表，未迁移时会提示先运行 `make setup` 或 `make migrate`。`make dev` 在一个终端启动 API、独立 Worker 和 frontend，控制台只显示地址与日志位置；三个服务的详细输出分别追加到 Git 忽略的 `artifacts/dev-logs/api.log`、`worker.log` 和 `frontend.log`，`Ctrl-C` 会一起停止。 默认地址： @@ -158,7 +158,7 @@ curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus API 为每个请求自行生成 `X-Request-ID` 并在响应中返回，不信任或回显客户端提供的同名 header。LLMBenchLab 生产日志调用只允许无格式参数的字面量消息；结构化 extra 除字段白名单外还逐字段执行固定枚举、UUID/Redis stream ID 与有限数值规范化，非法 ID 被省略，未知 method/code 只输出固定 `unsupported`。Redis Run 通知本身也只接受 canonical UUID。外部 logger 的动态消息不进入 JSON，原始 Uvicorn access handler 关闭。凭据和敏感内容仍绝不得放在 URL、header、请求路径或日志字段中。 -如需分别观察日志，可在三个终端运行 `make backend`、`make worker` 和 `make frontend`。只启动 API 时，新 Run 会持久化为 `pending`，但不会在 API 进程内执行。本地 SQLite 只支持一个 Worker；Redis URL 可留空，Worker 将使用数据库对账。所有命令可通过 `make help` 查看；更完整的环境变量、迁移和排障说明见 [`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md)。 +需要跟踪组合启动日志时可运行 `tail -f artifacts/dev-logs/api.log artifacts/dev-logs/worker.log artifacts/dev-logs/frontend.log`；需要在前台分别观察时，可在三个终端运行 `make backend`、`make worker` 和 `make frontend`。只启动 API 时，新 Run 会持久化为 `pending`，但不会在 API 进程内执行。本地 SQLite 只支持一个 Worker；Redis URL 可留空，Worker 将使用数据库对账。所有命令可通过 `make help` 查看；更完整的环境变量、迁移和排障说明见 [`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md)。 ## Mock Demo：完整离线流程 diff --git a/backend/tests/test_dev_script.py b/backend/tests/test_dev_script.py new file mode 100644 index 0000000..81845a8 --- /dev/null +++ b/backend/tests/test_dev_script.py @@ -0,0 +1,185 @@ +"""The combined development launcher keeps service chatter out of the console.""" + +from __future__ import annotations + +import os +import re +import stat +import subprocess +from pathlib import Path + +import pytest + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +DEV_SCRIPT = REPOSITORY_ROOT / "scripts" / "dev.sh" + + +def _write_executable(path: Path, contents: str) -> None: + path.write_text(contents, encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +def _prepare_project(tmp_path: Path, *, bootstrap_exit_code: int = 0) -> tuple[Path, Path]: + project = tmp_path / "project" + scripts = project / "scripts" + fake_bin = tmp_path / "bin" + scripts.mkdir(parents=True) + fake_bin.mkdir() + (project / "backend" / ".venv").mkdir(parents=True) + (project / "frontend" / "node_modules").mkdir(parents=True) + (scripts / "dev.sh").write_text(DEV_SCRIPT.read_text(encoding="utf-8"), encoding="utf-8") + + _write_executable( + scripts / "bootstrap_credential_keyring.sh", + f"""#!/usr/bin/env bash +echo 'keyring success status' +if (( {bootstrap_exit_code} != 0 )); then + echo 'keyring bootstrap failed safely' >&2 +fi +exit {bootstrap_exit_code} +""", + ) + _write_executable( + fake_bin / "fake-service", + """#!/usr/bin/env bash +set -euo pipefail +service="$1" +printf '%s child stdout\\n' "$service" +printf '%s child stderr\\n' "$service" >&2 +if [[ "${EXIT_SERVICE:-}" == "$service" ]]; then + sleep "${EXIT_DELAY_SECONDS:-0.2}" + exit "${EXIT_CODE:-0}" +fi +trap 'printf "%s:terminated\\n" "$service" >>"$EVENT_FILE"; exit 0' TERM INT +while true; do + sleep 0.1 +done +""", + ) + _write_executable( + fake_bin / "uv", + """#!/usr/bin/env bash +set -euo pipefail +if [[ " $* " == *" uvicorn "* ]]; then + exec "$FAKE_SERVICE_SCRIPT" api +fi +if [[ " $* " == *" app.worker "* ]]; then + exec "$FAKE_SERVICE_SCRIPT" worker +fi +echo 'unexpected fake uv invocation' >&2 +exit 91 +""", + ) + _write_executable( + fake_bin / "npm", + """#!/usr/bin/env bash +set -euo pipefail +exec "$FAKE_SERVICE_SCRIPT" frontend +""", + ) + return project, fake_bin + + +def _run_dev( + project: Path, + fake_bin: Path, + log_dir: Path, + *, + exit_service: str, + exit_code: int, +) -> subprocess.CompletedProcess[str]: + event_file = project / "service-events.log" + environment = os.environ.copy() + environment.update( + { + "PATH": f"{fake_bin}{os.pathsep}{environment['PATH']}", + "FAKE_SERVICE_SCRIPT": str(fake_bin / "fake-service"), + "EVENT_FILE": str(event_file), + "EXIT_SERVICE": exit_service, + "EXIT_CODE": str(exit_code), + "LLMBENCHLAB_DEV_LOG_DIR": str(log_dir), + } + ) + return subprocess.run( + ["bash", str(project / "scripts" / "dev.sh")], + cwd=project, + env=environment, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + +def test_dev_launcher_redirects_all_service_output_to_append_only_logs(tmp_path: Path) -> None: + project, fake_bin = _prepare_project(tmp_path) + log_dir = tmp_path / "custom-dev-logs" + log_dir.mkdir() + for service in ("api", "worker", "frontend"): + (log_dir / f"{service}.log").write_text("existing log entry\n", encoding="utf-8") + + result = _run_dev(project, fake_bin, log_dir, exit_service="api", exit_code=0) + + assert result.returncode == 0 + console = result.stdout + result.stderr + assert "Starting LLMBenchLab: Web http://127.0.0.1:5173 | API http://127.0.0.1:8000" in console + assert f"Logs: API {log_dir / 'api.log'}" in console + assert "API stopped with status 0" in console + assert "keyring success status" not in console + assert "child stdout" not in console + assert "child stderr" not in console + + for service in ("api", "worker", "frontend"): + contents = (log_dir / f"{service}.log").read_text(encoding="utf-8") + assert contents.startswith("existing log entry\n") + assert contents.count("=== LLMBenchLab dev session ") == 1 + assert re.search( + r"=== LLMBenchLab dev session \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z ===", + contents, + ) + assert f"{service} child stdout" in contents + assert f"{service} child stderr" in contents + assert stat.S_IMODE((log_dir / f"{service}.log").stat().st_mode) == 0o600 + assert stat.S_IMODE(log_dir.stat().st_mode) == 0o700 + + +def test_dev_launcher_propagates_failure_and_terminates_other_services(tmp_path: Path) -> None: + project, fake_bin = _prepare_project(tmp_path) + log_dir = tmp_path / "failure-logs" + + result = _run_dev(project, fake_bin, log_dir, exit_service="worker", exit_code=17) + + assert result.returncode == 17 + assert result.stdout.count("Starting LLMBenchLab:") == 1 + assert "child stdout" not in result.stdout + result.stderr + assert "child stderr" not in result.stdout + result.stderr + assert f"Error: Worker exited with status 17. Log: {log_dir / 'worker.log'}" in result.stderr + events = (project / "service-events.log").read_text(encoding="utf-8").splitlines() + assert set(events) == {"api:terminated", "frontend:terminated"} + assert all((log_dir / f"{service}.log").is_file() for service in ("api", "worker", "frontend")) + + +def test_dev_launcher_suppresses_keyring_success_but_preserves_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project, fake_bin = _prepare_project(tmp_path, bootstrap_exit_code=23) + log_dir = tmp_path / "unused-logs" + monkeypatch.setenv("PATH", f"{fake_bin}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("LLMBENCHLAB_DEV_LOG_DIR", str(log_dir)) + + result = subprocess.run( + ["bash", str(project / "scripts" / "dev.sh")], + cwd=project, + env=os.environ.copy(), + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + assert result.returncode == 23 + assert result.stdout == "" + assert "keyring success status" not in result.stderr + assert result.stderr.strip() == "keyring bootstrap failed safely" + assert not log_dir.exists() diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9547083..4aa3035 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -57,7 +57,17 @@ make setup make dev ``` -`scripts/dev.sh` 同时管理三个进程；任一进程退出会停止另外两个。需要分开查看日志时使用三个终端： +`scripts/dev.sh` 同时管理三个进程；任一进程退出会停止另外两个并传播其退出状态。组合启动的控制台只显示 Web/API 地址和日志位置，API、Worker、Vite 的 stdout/stderr 分别追加到 `artifacts/dev-logs/api.log`、`worker.log`、`frontend.log`，每次启动都有 UTC session marker。目录和日志在创建/复用时分别收紧为 `0700`/`0600`，且都被 Git 忽略；它们仍是敏感本地运维证据，不应上传。 + +跟踪详细输出可运行： + +```bash +tail -f artifacts/dev-logs/api.log \ + artifacts/dev-logs/worker.log \ + artifacts/dev-logs/frontend.log +``` + +需要把日志放到其他受保护目录时，仅为本地启动器设置 `LLMBENCHLAB_DEV_LOG_DIR`。需要让某个服务直接在前台输出时使用三个独立入口： ```bash make backend @@ -254,6 +264,7 @@ Pydantic 应用设置优先读取 `LLMBENCHLAB_*`，并为数据库、Redis、CO | `CORS_ORIGINS` / `LLMBENCHLAB_CORS_ORIGINS` | 两种 localhost `:5173` | 显式 allowlist；拒绝 `*` | | `FRONTEND_ORIGIN` | `http://localhost:5173` | 单 Origin 兼容别名；`CORS_ORIGINS` 优先 | | `LOG_LEVEL` / `LLMBENCHLAB_LOG_LEVEL` | `INFO` | `CRITICAL/ERROR/WARNING/INFO/DEBUG` | +| `LLMBENCHLAB_DEV_LOG_DIR` | `artifacts/dev-logs` | 仅供 `make dev` 使用的本地详细日志目录；不是应用 Settings 或生产日志后端 | | `LLMBENCHLAB_ENVIRONMENT` | `development` | `/info` 环境标签 | | `LLMBENCHLAB_DEBUG` | `false` | 只用于受控本地调试 | | `API_HOST` / `API_PORT` | `127.0.0.1` / `8000` | Make/脚本监听；Compose 只消费 host port | diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 4f29b41..d321ba1 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -22,6 +22,8 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 `0006` 兼容修复已完成本地 migration/full test/lint/Mock smoke、真实失败备份副本升级和当前重建库 startup/check；实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 4/4 成功，因此该维护任务为 `completed`，不改变 P2-07 的 `planned` 状态或范围。 +2026-08-30 的个人本地维护又从最新逻辑非空的 SQLite 一致性备份在 staging 中前进迁移并恢复 Demo 数据，同时让 `make dev` 默认把 API/Worker/Vite 详细输出分流到私有 Git 忽略日志。它只处理默认本地开发库和启动体验，不提供 PostgreSQL+keyring backup/restore、Redis rebuild 或告警演练，因而不算 P2-07 功能实现；本文件的下一切片保持不变。 + P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 ## 已完成：建立 P2-07 工作包 diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0c9ce5e..c1edcb8 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -331,6 +331,10 @@ PostgreSQL 不可用时没有可替代事实源。API 不应接受新 Run，Work 若恢复点早于某些 Provider 外发，数据库无法知道备份之后的远端消费；必须以 Provider 账单和 request ID 独立核对。Redis 可以重建通知，不能用来补造丢失的数据库事实。 +### 8.1 本地开发 SQLite 恢复边界 + +个人本地 SQLite 恢复不等同 P2-07 的 PostgreSQL/keyring 灾难恢复认证。恢复前必须停止 `make dev`、CLI 和任何直接连接者，并确认默认库没有进程占用或 WAL/SHM/journal sidecar。候选备份应按逻辑表计数、`PRAGMA quick_check`、外键、Alembic revision 和内容摘要选择，不能按文件体积猜测；含大量 freelist 的大文件可能在逻辑上为空。保留原候选不变，只在 staging 副本上执行受支持的 migration preflight/upgrade/check；迁移后数据摘要一致才可替换默认库。替换前另存并验证当前库，stored credential 还必须配对恢复数据库外 keyring，且核验过程不得输出 Key、ciphertext、nonce 或 keyring 内容。 + ## 9. Audit archive、核验与保留 `audit_events.expires_at` 表示行已达到最低保留期，不表示可由请求链路自动删除。Operational/security 事件仍分别至少保留 90/365 天；维护流程必须严格执行 **archive → offline verify → reconcile → maintenance-window delete**。Archive 是含内部 ID 的敏感运维文件，hash 只用于误改检测和精确文件确认，不是签名、认证、WORM 或不可篡改证明。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 8bc3f60..b23a997 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -57,10 +57,18 @@ - 真实失败备份副本已无损升级到 `0006` 并通过 integrity/FK/Alembic check；当前重建库由标准 preflight 自动备份后从 canonical `0005` 到 `0006`，startup gate、quick check、FK 和 metadata check 全部通过，业务计数与 Worker facts 保持。 - 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 的 backend、真实 PostgreSQL/Redis、完整 Compose 与 frontend 四个 job 全部成功；本维护任务完成，不改变 Phase 2/P2-07 状态。 +## 2026-08-30 本地数据恢复与静默启动（`in_progress`） + +- 默认本地 SQLite 重建库逻辑为空；多份约 96 MiB 候选也因 freelist 较大但业务表全空，不能恢复数据。最新且 revision 最高的非空一致性备份为 `backend/data/llmbenchlab.db.pre-alembic-20260827T073137431634Z.bak`（SHA-256 `7e046c1e7cd4ec39c5fe6f57b34f130670e0d249a70bf052a84a23e085a59a53`），含 1 个 Mock Model、1 个 Demo Benchmark、15 Questions、1 个 completed Run 和 15 Responses；与更早两份非空备份在全部共有列的内容摘要一致。 +- 停止 API/Worker/Vite 并确认默认库无占用/sidecar 后，只在该备份的一致性 staging 副本上执行 `0002→0006`。迁移前后五张旧表共有列摘要都为 `d1b3b74b7726f9e7903fbd3f445ad258d5f5aa4b885c976582f2d53e1d30302f`；恢复后的默认库 `quick_check=ok`、外键错误 0、Alembic current=head，API/Web 实际读取到 `1/1/1` 个 Model/Benchmark/Run。原始恢复源未修改；重建空库另存为 `backend/data/llmbenchlab.db.pre-original-data-restore-20260829T170121Z.bak`（SHA-256 `ec2ef8b2d5c9a338ce3e5f94c68a3c5742d288a798df2b7a6096960a48610c90`）。 +- `make dev` 现在只在控制台显示地址和日志位置；API、Worker、Vite 详细输出分别 append 到私有的 Git 忽略日志，单服务 Make 入口仍保留前台诊断。离线启动器 `3 passed`，真实本地启动/live/health/ready/API/Web 探针通过；完整 `make test` 为后端 `930 passed, 33 skipped`、前端 `38 passed`，lint/build/smoke/Compose config 也已通过。提交/push 与精确 SHA CI 尚在收尾，因此当前维护状态仍为 `in_progress`。 +- 该维护只恢复个人本地 SQLite Demo 数据并改进开发入口，不是 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建或灾难恢复认证；Phase 2/P2-07 状态不变。 + ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 - 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 +- 本地数据恢复与静默启动：状态为 `in_progress`；数据恢复和本地验证已完成，等待提交/push 与精确 SHA CI。 - P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 @@ -100,6 +108,7 @@ | P2-06 evidence 文档远程 CI | `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 run `33165775037` 4/4 | 精确文档 SHA 门禁完成；P2-06 为 `completed` | | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | +| 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；实现尚待 commit/push 与精确 SHA CI，不改变 P2-07 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | | P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | | P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | @@ -123,7 +132,8 @@ - [Phase 2 治理、审计与性能](worklogs/2026-08-27-phase-2-governance-audit-performance.md) - [Phase 2 正式 SLO 与容量模型](worklogs/2026-08-28-phase-2-slo-capacity-model.md) - [Phase 2 可观测性与审计保留](worklogs/2026-08-28-phase-2-observability-retention.md) +- [本地数据恢复与静默启动](worklogs/2026-08-30-restore-data-quiet-startup.md) ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。本次个人本地 SQLite/启动器维护不改变路线图：P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 继续保持 `in_progress`。 diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c671c16..9a6d58b 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -149,6 +149,7 @@ Web stored 流程如下： - 不记录 `os.environ`、HTTP headers、完整异常 locals、请求体或 `.env` 内容。 - 应用外的反向代理 access log、PostgreSQL/Redis server log、Docker/runtime log，以及 formatter 配置前或进程崩溃时的 stderr 不受上述 sanitizer 控制；未知且安装自有 handler 的第三方库也必须在部署侧审计、过滤并设置保留策略。 +- 本地 `make dev` 不再把三个服务的详细输出持续写到控制台，而是分别追加到 Git 忽略的 `artifacts/dev-logs/*.log`；启动器把目录/文件权限收紧为 `0700`/`0600`。Git ignore 和文件权限都不是加密、自动保留或安全删除，日志仍不得包含 Key、请求体或完整 Provider 内容，也不得作为公开附件上传。 - 即使进程内原始 Uvicorn access handler 已禁用，秘密仍不得出现在 URL、查询参数或路径；反向代理和基础设施日志必须应用相同规则。 - `DEBUG` 仅用于无秘密的本地排障。HTTP wire logging 和第三方 SDK debug logging 默认关闭；即使 SQL 参数已隐藏，也不应在含敏感数据的部署中采集冗长 SQL trace。 - 原始模型回答可能含供应商回显的其他敏感内容。当前只保证精确替换正在使用的 Key，并识别部分常见秘密形态；回答仍会作为评测证据持久化并通过 Responses API 返回，不能把“没有当前 Key”误当成“可以公开”。 diff --git a/docs/TESTING.md b/docs/TESTING.md index 895db9e..8093df5 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -15,7 +15,7 @@ LLMBenchLab 的默认验收路径必须完全离线、可重复且不产生模 | API 与进程边界测试 | FastAPI Schema、状态码、秘密安全、Run 提交与 API/Worker 分离 | `backend/tests/test_api.py`、`test_run_dispatch.py`、`test_process_boundaries.py` | 禁止 | | Governance / audit 测试 | policy/ledger 完整性、typed history/audit、Provider/credential evidence，以及 canonical archive/离线 verify/精确 reconcile/restore/delete | `test_governance.py`、`test_audit_api.py`、`test_audit_archive.py`、`test_audit_retention.py`、`test_audit_retention_cli.py` | 禁止；SQLite/Mock/fixture；真实 PG retention 只连测试库 | | 租约与 Worker 测试 | 条件领取、fencing、心跳、取消、幂等 Response、重试/恢复、队列 ACK，以及 generation 级 DB-time scan/claim/lease-heartbeat/progress/stale | `test_run_leases.py`、`test_evaluation_runner_reliability.py`、`test_worker.py`、`test_worker_progress.py`、`test_worker_probe.py` | 禁止；SQLite/假队列 | -| Metrics / alert / logging 测试 | 固定 Prometheus exposition、snapshot/hard cap/single-flight/取消竞态、精确八规则/Runbook，以及全部生产 logger source/第三方 handler 治理 | `test_prometheus_exporter.py`、`test_prometheus_alert_rules.py`、`test_logging.py`、`test_logging_sources.py` | 禁止；SQLite/假队列/标准库 JSON | +| Metrics / alert / logging 测试 | 固定 Prometheus exposition、snapshot/hard cap/single-flight/取消竞态、精确八规则/Runbook、全部生产 logger source/第三方 handler 治理，以及组合开发启动器的日志分流/退出传播 | `test_prometheus_exporter.py`、`test_prometheus_alert_rules.py`、`test_logging.py`、`test_logging_sources.py`、`test_dev_script.py` | 禁止；SQLite/假队列/标准库 JSON/假子进程 | | 迁移与导入回归 | SQLite/真实 PostgreSQL migration、`0005` populated downgrade refusal/空库往返，以及 13 表 SQLite→PostgreSQL 原子导入 | `test_migrations.py`、`test_sqlite_postgres_import.py` | 导入/本地部分禁止；真实 PostgreSQL 用 `integration` marker | | 真实基础设施集成 | PostgreSQL 并发领取/取消竞态、Redis Streams PEL/ACK/重复投递 | `backend/tests/integration/` 与 importer 的 `integration` 用例 | 只连接显式测试 PostgreSQL/Redis；禁止 Provider | | Mock 端到端 Smoke | API 提交 pending Run → 独立 WorkerService → Responses → Leaderboard/Metrics | `backend/tests/test_smoke.py`，marker 为 `smoke` | 禁止 | @@ -470,7 +470,7 @@ keyring bootstrap 的 `24` 个定向测试覆盖所有相关本地入口强制 C make dev ``` -它同时启动 API、独立 Worker 和 Vite。若拆分终端，必须分别运行 `make backend`、`make worker` 与 `make frontend`；只启动 API 时新 Run 保持 `pending` 是预期行为。 +它同时启动 API、独立 Worker 和 Vite。正常运行期间控制台应只保留启动地址和三个详细日志路径；API 请求、Worker 和 Vite 输出写入 `artifacts/dev-logs/*.log`。若拆分终端，必须分别运行 `make backend`、`make worker` 与 `make frontend`；只启动 API 时新 Run 保持 `pending` 是预期行为。`backend/tests/test_dev_script.py` 使用假 `uv`/`npm` 验证 stdout/stderr 分流、append、`0700/0600` 权限、失败码传播和其余进程清理，不启动真实服务。 ### 11.1 API 验收 diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index f155ed4..2ffea0b 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -71,6 +71,8 @@ `slice_delivered` 表示该垂直切片及其候选门禁已交付，不表示整个阶段完成。Phase 2 必须保持 `in_progress`。 +2026-08-30 的个人本地维护已从最新非空 SQLite 一致性备份恢复 1 个 Mock Model、1 个 Demo Benchmark、15 Questions、1 个 completed Run 和 15 Responses，并让组合开发启动器把三服务详细输出分流到私有 Git 忽略日志。该操作有 staging 迁移、共有列摘要、完整性/外键/head、真实本地启动与 API/Web 读取证据，但不包含 PostgreSQL/keyring、Redis 或告警恢复认证，不能计作 P2-07 实施或改变本阶段状态。 + ## 验收标准与当前结论 - [x] 可靠执行基础：API/Worker restart、真实 lease-owner `SIGKILL`、Redis stop/start、duplicate delivery、pending/running cancel 和 lease takeover 有历史真实 PostgreSQL/Redis/Compose 证据。 @@ -113,6 +115,7 @@ | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | | P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | | 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | `8fb51b6…` 的 exact-SHA run `33263405214` 4/4；不改变 P2-07 planned 状态 | +| 2026-08-30 本地恢复/静默启动维护 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、SQLite digest/quick/FK/head、真实 API/Web 读取通过 | 个人本地 Demo 数据恢复和开发 UX；尚待精确 SHA CI，且不是 P2-07 恢复认证 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 diff --git a/docs/plans/2026-08-30-restore-data-quiet-startup.md b/docs/plans/2026-08-30-restore-data-quiet-startup.md new file mode 100644 index 0000000..7716afc --- /dev/null +++ b/docs/plans/2026-08-30-restore-data-quiet-startup.md @@ -0,0 +1,106 @@ +# 恢复本地数据并静默一键启动执行计划 + +- Owner: Codex +- Status: active +- Created: 2026-08-30 +- Updated: 2026-08-30 +- Related phase: [Phase 2](../phases/PHASE-2-RELIABILITY.md) +- Worklog: [2026-08-30-restore-data-quiet-startup](../worklogs/2026-08-30-restore-data-quiet-startup.md) +- ADRs: 无；本次不改变数据库协议或应用日志安全合同，只恢复既有本地数据并调整开发启动器的输出去向。 + +## Context + +当前默认 SQLite `backend/data/llmbenchlab.db` 已在上次兼容修复中重建到 Alembic head，但业务表为空。开发启动器 `scripts/dev.sh` 正在同时运行 API、Worker 与 Vite，并把三个子进程的全部 stdout/stderr 混合输出到当前控制台。仓库中存在多份迁移前 SQLite 一致性备份；必须从逻辑行数、完整性和 revision 识别真正含原数据的来源，不能仅按文件大小猜测。 + +## Objective + +在保留当前新库和所有既有备份的前提下，把最近一份可验证的原始业务数据恢复到默认 SQLite 并迁移到当前 head；随后让 `make dev` 只显示简洁启动/退出状态，把三个服务的详细日志分别写入 Git 忽略的本地日志文件。 + +## Scope + +- 只读核验候选 SQLite 备份、keyring 存在性和当前数据库占用者。 +- 停止当前本地开发进程，制作可恢复的当前库快照，迁移旧库副本并原子替换默认 SQLite。 +- 修改 `scripts/dev.sh` 的日志重定向、生命周期监管和失败提示；单独的 `make backend`、`make worker`、`make frontend` 保持流式诊断行为。 +- 增加启动器回归测试并同步本地运行、恢复、安全和项目状态文档。 + +## Non-goals + +- 不新增通用备份产品、日志轮转守护进程、日志聚合系统或生产依赖。 +- 不改变应用 JSON 日志格式、级别、API、Benchmark 协议、数据库 schema 或迁移历史。 +- 不恢复 `backend/data` 以外的数据库，不删除任何候选备份，不调用真实 Provider。 + +## Assumptions + +- “原来的数据”指仓库默认本地 SQLite 在重建为空库前保存的业务数据；以候选备份的逻辑行数、最新时间、完整性和迁移可达性验证。 +- 当前 `make dev` 的 API/Worker/Vite 是唯一默认库写入者；通过文件占用和端口检查确认并在替换前停止。 +- 详细日志仍需可追查，因此写入本地 Git 忽略文件，而不是丢弃到 `/dev/null`。 + +## Requirements + +- 遵守 `AGENTS.md` 的数据保护、迁移前备份、离线 Mock 测试、文档联动、阶段提交/push 和精确 SHA CI 门禁。 +- 恢复后必须满足 SQLite `quick_check=ok`、零外键错误、Alembic current=head，且业务表计数与恢复源一致。 +- `make dev` 正常运行期间控制台不得持续刷出 API/Worker/Vite 详细日志；子进程失败时必须返回非零并指出对应日志文件。 + +## Implementation steps + +1. [completed] 核验备份、占用进程和启动器现状，确定唯一恢复源与最小输出行为。 + - Files/modules: `backend/data/*.bak`、`.secrets/credential-keys.json`（仅元数据）、`scripts/dev.sh`、相关文档/测试 + - Validation: 候选 revision、完整性、业务计数和文件占用有可复核结果，恢复源选择有明确理由。 +2. [completed] 停止本地服务，在同目录准备并迁移恢复副本，验证后以可回滚的 rename 替换默认库。 + - Files/modules: Git 忽略的 `backend/data/llmbenchlab.db*` + - Validation: 当前新库保留为明确命名备份；恢复后 head、完整性、外键与逻辑计数全部通过。 +3. [completed] 实现静默 `make dev` 并增加回归测试。 + - Files/modules: `scripts/dev.sh`、`backend/tests/test_dev_script.py` + - Validation: 正常启动只输出简洁摘要，三个日志文件接收详细输出，失败状态和清理行为有自动化覆盖。 +4. [in_progress] 运行定向与全量验证，更新强制文档并完成审查、提交、push 与精确 SHA CI。 + - Files/modules: README、部署/运维/安全/测试、CHANGELOG、PROJECT_STATUS、PHASE-2、NEXT_TASK、本计划与工作日志 + - Validation: 本地验证通过、diff/秘密扫描通过、远程必需 jobs 对精确 SHA 全绿。 + +## Risks + +| 风险 | 可能性/影响 | 预防措施 | 触发后的处理 | +|---|---|---|---| +| 选择到逻辑空备份 | 中/高 | 比较所有候选的表计数、revision、完整性和时间，不按体积选择 | 不替换当前库，继续保留并核对候选 | +| 活跃进程在替换时写库 | 已发生/高 | 替换前识别并停止占用默认库的 API/Worker，复核无 WAL/SHM 写入者 | 放弃替换并从保留快照回滚 | +| 旧数据迁移到 head 失败或漂移 | 低/高 | 只迁移同目录 staging 副本，运行受支持 preflight、upgrade、check 和内容对账 | 保持当前库不变，保存失败副本供诊断 | +| stored credential 与 keyring 不匹配 | 低/高 | 只检查 keyring 元数据/权限和 credential 行数，不输出任何密钥或 envelope | 数据可恢复但 stored 模式保持 fail closed，并明确记录需重新输入 Key | +| 日志静默掩盖启动失败 | 中/中 | 保留每服务独立日志，父进程返回子进程状态并显示日志路径 | 用户按提示查看对应日志；单服务 Make 入口仍流式输出 | + +## Validation + +| 验收项 | 命令或检查 | 预期结果 | 实际结果 | +|---|---|---|---| +| 恢复源可信 | SQLite 只读 revision/count/quick/FK/digest 核验 | 最新含数据候选完整且可迁移 | 通过：选定最新非空 `0002`，quick/FK 正常且与更早非空备份共有列摘要一致 | +| 恢复后数据一致 | SQLite count/digest、`alembic current/check` | 业务数据与源一致，revision 为 head | 通过：`1/1/15/1/15`，共有列 digest 一致，quick/FK/head/check 全绿 | +| 静默启动行为 | 定向 pytest + 有界 `make dev` 探针 | 控制台简洁、详细日志分流、服务可用 | 通过：`3 passed`；真实启动期间仅两行摘要，API/Web/恢复数据可读，详细日志 `0600` | +| 全量质量门禁 | `make test`、`make lint`、frontend build、`make smoke`、Compose config | 全部适用项通过 | 通过：backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config 全绿，build 保留既有 chunk warning | +| 远程门禁 | push 后 GitHub Actions | 精确提交 4/4 必需 job 成功 | 待执行 | + +## Rollback + +数据库替换前已通过 SQLite backup API 把当前默认库保存为带 UTC 时间戳的同目录 `.bak`，且没有覆盖任何既有文件；若后验验证失败，停止服务后将恢复库另存，再由该备份恢复默认路径。代码回滚只需恢复 `scripts/dev.sh`；已有日志文件为 Git 忽略的本地诊断资产，不参与数据库事实或协议。 + +## Documentation updates + +- [x] README / 用户操作说明 +- [x] API / 数据格式 / Benchmark 协议（不适用：接口与协议不变） +- [x] Deployment / Operations / Security / Testing（恢复与日志边界） +- [x] CHANGELOG、PROJECT_STATUS、阶段文档、NEXT_TASK、工作日志 + +## Completion evidence + +- Changed files: `scripts/dev.sh`、`backend/tests/test_dev_script.py`、README、Deployment/Operations/Security/Testing、CHANGELOG、PROJECT_STATUS、Phase 2、NEXT_TASK、本计划与工作日志；另有 Git 忽略的默认 SQLite 恢复和安全备份。 +- Commands run: SQLite 只读审计/backup/staging migration/digest/head 检查；启动器定向 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/Compose config；真实有界 `make dev` 与 API/Web 探针。 +- Acceptance evidence: 默认库 `1/1/15/1/15` 且共有列 digest 与源一致；quick/FK/head/check 通过；真实运行控制台仅两行摘要，日志分流为 `0600`，API/Web 可读恢复数据。 +- Not run: 本地真实 PostgreSQL/Redis integration、capacity/SLO/Phase 2 acceptance；本任务不改变其语义，远程必需 jobs 尚待提交后运行。 +- Known issues: 既有 Python 3.14 依赖 warning 和 Vite 662.39 kB chunk warning；2026-08-28 大备份逻辑为空但继续保留。 + +## Decision and discovery log + +| 日期 | 类型 | 记录 | 影响/后续 | +|---|---|---|---| +| 2026-08-30 | discovery | 当前默认库和三份 96 MiB 迁移前备份业务表均为 0 行；2026-08-25/27 的较小备份仍含 1 Model、1 Benchmark、15 Questions、1 Run 和 15 Responses。 | 恢复源必须从较小含数据备份中按最新 revision/时间进一步核验。 | +| 2026-08-30 | discovery | 当前 API、Worker 和 Vite 仍在运行，API/Worker 持有默认 SQLite 文件。 | 恢复步骤开始前必须停止该开发进程组并再次复核占用。 | +| 2026-08-30 | decision | 选择 `backend/data/llmbenchlab.db.pre-alembic-20260827T073137431634Z.bak`：它是最新、revision 最高且逻辑非空的候选，SHA-256 为 `7e046c1e7cd4ec39c5fe6f57b34f130670e0d249a70bf052a84a23e085a59a53`；与两个更早非空备份在所有共有列上的内容摘要一致。 | 只在该文件的一致性副本上迁移；源文件保持只读不变。 | +| 2026-08-30 | discovery | staging 从 `0002` 顺序升级到 `0006` 后，五张原有业务表共有列摘要保持 `d1b3b74b7726f9e7903fbd3f445ad258d5f5aa4b885c976582f2d53e1d30302f`；完整性、外键、head、索引和 active-state 检查通过。 | 允许替换默认库。 | +| 2026-08-30 | action | 当前空库先通过 SQLite backup API 保存为 `backend/data/llmbenchlab.db.pre-original-data-restore-20260829T170121Z.bak`，SHA-256 `ec2ef8b2d5c9a338ce3e5f94c68a3c5742d288a798df2b7a6096960a48610c90`；随后用已验证 staging 同目录替换默认库。 | 空库及其 3 条 Worker process facts 可回滚；原恢复源未修改。 | diff --git a/docs/worklogs/2026-08-30-restore-data-quiet-startup.md b/docs/worklogs/2026-08-30-restore-data-quiet-startup.md new file mode 100644 index 0000000..d96333f --- /dev/null +++ b/docs/worklogs/2026-08-30-restore-data-quiet-startup.md @@ -0,0 +1,159 @@ +# 2026-08-30 — 恢复本地数据并静默一键启动工作日志 + +> 本日志记录实际发生的工作，不是事后美化的总结。所有命令以仓库根目录为基准。 + +## 元信息 + +- 日期：2026-08-30 +- 执行者：Codex +- 关联阶段：[Phase 2](../phases/PHASE-2-RELIABILITY.md) +- 关联计划：[恢复本地数据并静默一键启动](../plans/2026-08-30-restore-data-quiet-startup.md) +- 关联 ADR：无；不改变 schema、协议或应用日志合同 +- 最终状态：in_progress + +## 初始仓库状态 + +- 当前分支：`codex/complete-evaluation-workflow`，跟踪 `origin/codex/complete-evaluation-workflow` +- `git status --short --branch` 摘要：任务开始时工作树干净 +- 已有未提交改动：无；所有 SQLite/keyring/log 文件均为 Git 忽略的本地资产，恢复时不删除既有备份 +- 相关功能与测试现状：默认 SQLite 已迁移到 `20260829_0006`，但核心业务表均为 0 行；`scripts/dev.sh` 把 API、Worker、Vite 的 stdout/stderr 直接混合到控制台 +- 环境约束：macOS 本地工作区；当前 `make dev` 进程占用默认 SQLite 并监听 `127.0.0.1:8000`、`127.0.0.1:5173`；禁止真实 Provider 调用和秘密输出 + +## 本次目标与背景 + +用户重建数据库后可以启动项目，但原有本地数据不可见，同时一键启动在控制台持续输出大量服务日志。本次先恢复最近可验证的原数据，再让 `make dev` 保持简洁控制台输出且仍可从本地文件排障，然后停止，不扩展其他功能。 + +## 范围 + +- 核验并恢复默认本地 SQLite 数据，保留当前新库和所有旧备份。 +- 调整 `scripts/dev.sh`，把三个服务详细日志分别写入 Git 忽略的本地文件。 +- 增加最小自动化回归并同步强制文档。 + +## 非目标 + +- 不新增生产备份、日志采集/轮转基础设施或依赖。 +- 不修改 API、Benchmark 协议、schema、应用 JSON 日志内容或单服务调试入口。 +- 不删除任何数据库、备份、keyring 或用户 artifact。 + +## 验收标准 + +- [x] 默认 SQLite 恢复到 Alembic head，`quick_check` 为 `ok`、无外键错误，业务表计数/内容摘要与选定恢复源一致。 +- [x] 当前重建空库以明确 UTC 时间戳备份保留，原始候选备份不被修改。 +- [x] `make dev` 正常运行时不持续刷 API/Worker/Vite 详细日志，并清楚显示服务地址和日志位置。 +- [x] 任一子进程退出仍会停止其余进程；失败时返回非零并指向详细日志。 +- [x] 定向/全量测试、lint、build、离线 smoke 和适用部署检查有真实结果，文档与项目状态同步。 +- [ ] 阶段提交 push 到工作分支，对应精确 SHA 的 GitHub Actions 必需 jobs 全部成功。 + +## 假设 + +- “原来的数据”是默认 SQLite 重建前的最近逻辑非空备份；通过时间、revision、计数、完整性和迁移验证确定。 +- 详细日志不能丢弃，默认写入仓库内 Git 忽略目录；单独的 Make 服务入口仍用于实时诊断。 + +## 风险 + +| 风险 | 影响 | 缓解措施 | 结果 | +|---|---|---|---| +| 恢复错库或空库 | 原数据仍不可见或被错误替换 | 全量枚举候选并比较逻辑计数、revision、时间和完整性 | 已选定最新非空 0002，API/Web 可读恢复数据 | +| 活跃写入导致不一致 | SQLite 损坏或丢写 | 停止占用者、同目录 staging、替换前后完整性检查 | 替换窗口无占用/sidecar，后验通过 | +| 迁移破坏旧数据 | 恢复数据漂移 | 只在副本迁移，保留源和当前库，按表计数/摘要对账 | 共有列 digest 前后一致，源与空库备份均保留 | +| 日志改动隐藏错误 | 启动失败难诊断 | 独立日志文件、退出码传播、简洁失败提示 | fake failure/cleanup 和真实启动探针通过 | + +## 实施步骤 + +1. [completed] 只读核验候选备份、当前进程、keyring 边界和启动脚本。 +2. [completed] 停止服务、准备/迁移/验真副本并可回滚替换默认库。 +3. [completed] 实现静默启动与回归测试。 +4. [in_progress] 运行完整验证、更新文档、审查、提交、push 并等待精确 SHA CI。 + +## 实际修改 + +| 文件/模块 | 修改内容 | 对应需求/原因 | +|---|---|---| +| `docs/plans/2026-08-30-restore-data-quiet-startup.md` | 建立可持续执行计划、恢复回滚和验收门禁 | 数据恢复跨持久层与启动器，需要明确保护边界 | +| `docs/worklogs/2026-08-30-restore-data-quiet-startup.md` | 记录初始状态、范围、风险与真实执行证据 | 仓库强制任务追踪 | +| Git 忽略的 `backend/data/llmbenchlab.db` | 从最新非空 0002 一致性备份副本迁移到 0006 后替换默认空库 | 恢复用户原有本地数据 | +| `scripts/dev.sh` | 三服务 stdout/stderr 分流到私有本地日志，控制台保留简洁摘要，失败状态继续传播 | 启动时不再持续刷屏且仍可排障 | +| `backend/tests/test_dev_script.py` | 用假进程覆盖日志 append/权限/分流、失败码和清理 | 新启动行为的离线回归 | +| `README.md`、`docs/DEPLOYMENT.md` | 说明安静组合启动、日志路径、跟踪方式和 override | 用户操作方式变化 | +| `docs/SECURITY.md`、`docs/OPERATIONS.md`、`docs/TESTING.md` | 记录私有日志安全边界、本地 SQLite 恢复纪律和测试覆盖 | 安全/恢复/验证联动 | +| `CHANGELOG.md`、`docs/PROJECT_STATUS.md`、`docs/phases/PHASE-2-RELIABILITY.md`、`docs/NEXT_TASK.md` | 同步已验证事实并明确不改变 P2-07/Phase 2 状态 | 仓库强制收尾 | + +## 决定、偏差与发现 + +| 时间 | 类型 | 事实与理由 | 后续影响 | +|---|---|---|---| +| 00:xx CST | discovery | 默认库、两份 2026-08-28 备份及 2026-08-29 空库备份的业务表均为 0 行，不能按 96 MiB 文件大小认定含原数据。 | 继续核验更早且逻辑非空的候选。 | +| 00:xx CST | discovery | 2026-08-25/27 的三个迁移前备份均含 1 Model、1 Benchmark、15 Questions、1 Run、15 Responses；最新一份 revision 为 `20260825_0002`。 | 将对最新非空候选做内容摘要、状态与迁移演练。 | +| 00:xx CST | discovery | API/Worker/Vite 当前仍运行并占用默认库。 | 数据替换前停止整个本地开发进程组。 | +| 01:01 CST | decision | 选定最新非空且 revision 最高的 `backend/data/llmbenchlab.db.pre-alembic-20260827T073137431634Z.bak`；其 SHA-256 为 `7e046c1e7cd4ec39c5fe6f57b34f130670e0d249a70bf052a84a23e085a59a53`，与更早两份备份的共有列内容摘要一致。 | 原文件不修改，只复制到 staging 并向前迁移。 | +| 01:01 CST | action | 向已确认的开发进程组发送 SIGINT，父启动器按既有 cleanup 停止 API、Worker、Vite；随后文件占用、8000/5173 listener 与 WAL/SHM/journal 复核均为空。 | 默认 SQLite 已进入可安全离线替换窗口。 | +| 01:02 CST | validation | 首次 staging 后验 SQL 误把 reservation 列写成 `status`，SQLite 以 `no such column: status` 安全失败；检查 schema 后改用真实列 `state` 重跑。 | 该失败只发生在只读核验语句，未修改 staging；已在下条验证中纠正。 | +| 01:02 CST | validation | staging 完成 `0002→0003→0004→0005→0006`；五张旧业务表共有列摘要保持 `d1b3b74b7726f9e7903fbd3f445ad258d5f5aa4b885c976582f2d53e1d30302f`，quick/FK/head/三个修复索引通过，active Run/reservation 为 0。 | 证明迁移未改变原有业务内容且可离线替换。 | +| 01:03 CST | action | 当前空库通过 SQLite backup API 保存为 `backend/data/llmbenchlab.db.pre-original-data-restore-20260829T170121Z.bak`，SHA-256 `ec2ef8b2d5c9a338ce3e5f94c68a3c5742d288a798df2b7a6096960a48610c90`，quick/FK/head 均通过；已验证 staging 随后同目录替换默认库。 | 默认库恢复原数据；空库仍可恢复，源备份未改。 | +| 01:04 CST | implementation | 组合启动器保留应用 INFO 日志合同，把 API/Worker/Vite stdout/stderr 分别 append 到本地日志；控制台只显示地址/路径，子进程退出仍传播状态并清理其余服务。 | 不通过提高全局日志级别或丢弃 stderr 隐藏故障；单服务入口不变。 | +| 01:05 CST | validation | 首次从 `backend/` 运行定向组合命令时误写 `scripts/dev.sh`，`bash -n`/ShellCheck 因路径不存在失败；该命令未启用 `set -e`，后续 Ruff/pytest 仍通过。随后用 `set -euo pipefail` 和 `../scripts/dev.sh` 完整重跑，全项通过。 | 最终证据只采用纠正后的完整命令，同时保留首次操作偏差。 | +| 01:06 CST | validation | 真实 `make dev` 控制台只出现两行摘要；live/health/ready、Web 及 Model/Benchmark/Run 读取通过，重复请求后控制台无新增输出。发送 Ctrl-C 后 Make 报预期中断 130，所有服务/端口退出。 | 用户可正常启动且不再刷屏；详细日志仍可追查。 | +| 01:11 CST | review | 独立终审为 0 Blocker/High/Medium；指出 PROJECT_STATUS 仍写“等待完整测试”的 Low 文档残留，已改为只等待提交/push/CI。可信本地日志目录的同用户 symlink 边界为非阻断 Low，默认目录 `0700` 且威胁模型不承诺防同权限攻击者，本次不扩成安全文件安装器。 | 关闭文档不一致，保持最小范围。 | + +## 实际运行命令 + +| 命令 | 目的 | 退出码 | 结果摘要 | +|---|---|---:|---| +| `git status --short --branch` | 确认分支和用户改动 | 0 | 工作树初始干净 | +| `sqlite3 -readonly <候选库> ...` | 核验 revision、quick check、外键和业务表计数 | 0 | 96 MiB 候选逻辑为空；三个早期候选逻辑非空且完整 | +| `lsof -nP backend/data/llmbenchlab.db` 及端口检查 | 识别数据库占用者 | 0 | API/Worker 持有 DB；API/Vite 分别监听 8000/5173 | +| `kill -INT -- -67299` 后复核文件/端口占用 | 优雅停止当前 `make dev` 进程组 | 0 | API、Worker、Vite 全部停止；默认库无占用、无 sidecar | +| staging SQLite backup + `prepare_migrations` + `alembic upgrade head/check/current` | 在副本上迁移原数据 | 0 | `0002→0006` 成功，Alembic 无待生成操作，current 为 head | +| staging 首次 active reservation 核验 | 检查迁移后无活跃外发状态 | 1 | 只读 SQL 使用了不存在的 `status` 列；未修改 DB，按 schema 改为 `state` | +| staging/common-column digest、quick/FK/count/index 核验 | 证明数据和 schema 可替换 | 0 | 摘要一致；`1/1/15/1/15`，active 为 0，三个修复索引存在 | +| SQLite backup API + 默认库替换 + 后验 digest/quick/FK/count/head | 可回滚恢复原数据 | 0 | 默认库已为 0006；核心计数 `1/1/15/1/15`，1 个 completed Run、15 Responses | +| `make migrate` | 用仓库受支持入口复核恢复后的默认库 | 0 | no-op 成功 | +| `bash -n scripts/dev.sh` 等（cwd=`backend`） | 首次定向检查 | 部分失败 | 路径写错导致 shell syntax/ShellCheck 未运行成功；Ruff 与 `3 passed` 继续执行，随后整体纠正重跑 | +| `set -euo pipefail; bash -n ../scripts/dev.sh; shellcheck ../scripts/dev.sh; uv run ruff ...; uv run pytest -q tests/test_dev_script.py` | 静默启动定向回归 | 0 | shell/Ruff/format 通过；`3 passed`，仅既有依赖 warning | +| 有界 `make dev` + curl/live/API/Web + Ctrl-C | 真实本地启动与恢复数据读取 | 预期中断 | 运行期间控制台仅两行；live/health/ready 正常，Web 200，Models/Benchmarks/Runs=`1/1/1`，日志 `0600`；退出后无占用 | +| `make test` | 完整离线回归 | 0 | backend `930 passed, 33 skipped`；frontend 9 files / `38 passed`；warning 均为既有依赖/迁移负向用例 | +| `make lint` | Ruff/format/ESLint/TypeScript | 0 | Ruff 154 files、ESLint、typecheck 全部通过 | +| `cd frontend && npm run build` | production build | 0 | 2192 modules，构建成功；保留既有主 chunk 662.39 kB warning | +| `make smoke` | 完全离线 Mock 垂直链路 | 0 | `1 passed, 7 deselected` | +| `docker compose config --quiet` | 部署配置语法 | 0 | 通过，未启动容器 | +| `alembic current && alembic check` + 默认库 quick/FK/count | 恢复后最终 schema/data 复核 | 0 | `0006 (head)`、无新操作、quick ok、FK 0、`1/1/15/1/15` | +| 真实启动后的共有列 digest/Worker process 核验 | 确认启动探针未改历史业务数据 | 0 | digest 仍为 `d1b3b74b…0302f`；新增 1 条正常 stopped Worker process，active 为 0 | + +## 测试结果 + +- 通过：启动器 `3 passed`；完整后端 `930 passed`，前端 `38 passed` +- 失败：最终验证 0；曾有一次只读 reservation 列名错误和一次检查路径错误，均已记录并完整纠正重跑 +- Lint/typecheck/build：Ruff/format 154 files、ESLint、TypeScript、Vite build 全部通过；保留既有 662.39 kB chunk warning +- Smoke/Docker：离线 Mock smoke `1 passed, 7 deselected`；Compose config 通过且未启动容器 + +## 未运行验证 + +- 真实 PostgreSQL/Redis integration、capacity、SLO 和 Phase 2 acceptance：未运行；本次只修改本地启动器和个人 SQLite 数据，不改变其生产/并发语义，远程 CI 仍会执行仓库必需 integration/full-stack jobs。 + +## 未完成项 + +- 数据库恢复、代码、文档和本地全量验证已完成；仍需终审、提交/push 与精确 SHA CI。 + +## 已知问题与限制 + +- 2026-08-28 的两份 96 MiB 备份逻辑为空且主要为空闲页，保留但未作未授权取证；本次恢复的是最新可验证的逻辑非空备份。 + +## 安全检查 + +- 真实密钥扫描：未暂存 diff 的高置信模式扫描和 `git diff --check` 已通过；提交前仍复核 staged diff；核验未读取或输出 keyring 内容、credential envelope 或 Provider Key +- 真实 API 调用：否 +- 日志/API 脱敏：应用日志合同不变；组合启动详细输出保留到 Git 忽略目录，目录/文件收紧为 `0700/0600` +- 危险 Git 操作（force push/reset 等）：无 +- 阶段 push：待执行 +- 远程 CI：待执行 +- 遗留安全风险：数据库与 keyring 必须配对恢复；若存在 stored credential 且 keyring 不匹配，应保持 fail closed + +## 结果与下一步 + +进行中；数据、实现、文档和本地门禁已经完成，下一步是终审、staged diff/秘密复核、commit/push 和精确 SHA CI。 + +## 最终 Git 状态 + +```text +任务进行中 +``` diff --git a/scripts/dev.sh b/scripts/dev.sh index 3d162ca..fe3958e 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -16,20 +16,31 @@ if [[ ! -d backend/.venv || ! -d frontend/node_modules ]]; then exit 1 fi -./scripts/bootstrap_credential_keyring.sh +./scripts/bootstrap_credential_keyring.sh >/dev/null if [[ -f .env ]]; then set -a # shellcheck disable=SC1091 source ./.env set +a -else - echo "Notice: .env is absent; safe local defaults will be used. Run 'make setup' to create it." fi api_host="${API_HOST:-127.0.0.1}" api_port="${API_PORT:-8000}" frontend_host="${FRONTEND_HOST:-127.0.0.1}" +dev_log_dir="${LLMBENCHLAB_DEV_LOG_DIR:-$project_root/artifacts/dev-logs}" +mkdir -p "$dev_log_dir" +chmod 700 "$dev_log_dir" + +api_log="$dev_log_dir/api.log" +worker_log="$dev_log_dir/worker.log" +frontend_log="$dev_log_dir/frontend.log" +session_started_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +for service_log in "$api_log" "$worker_log" "$frontend_log"; do + touch "$service_log" + chmod 600 "$service_log" + printf '\n=== LLMBenchLab dev session %s ===\n' "$session_started_at" >>"$service_log" +done backend_pid="" worker_pid="" @@ -56,47 +67,64 @@ cleanup() { trap cleanup EXIT trap 'exit 130' INT TERM -echo "Starting API at http://${api_host}:${api_port}" +printf 'Starting LLMBenchLab: Web http://%s:5173 | API http://%s:%s\n' \ + "$frontend_host" "$api_host" "$api_port" +printf 'Logs: API %s | Worker %s | frontend %s\n' \ + "$api_log" "$worker_log" "$frontend_log" ( cd backend exec uv run uvicorn app.main:app --host "$api_host" --port "$api_port" --reload -) & +) >>"$api_log" 2>&1 & backend_pid=$! -echo "Starting independent Worker" ( cd backend exec uv run python -m app.worker -) & +) >>"$worker_log" 2>&1 & worker_pid=$! -echo "Starting web app at http://${frontend_host}:5173" ( cd frontend exec npm run dev -- --host "$frontend_host" -) & +) >>"$frontend_log" 2>&1 & frontend_pid=$! -status=0 +stopped_service="" +stopped_pid="" +stopped_log="" while true; do if ! kill -0 "$backend_pid" 2>/dev/null; then - wait "$backend_pid" || status=$? + stopped_service="API" + stopped_pid="$backend_pid" + stopped_log="$api_log" break fi if ! kill -0 "$worker_pid" 2>/dev/null; then - wait "$worker_pid" || status=$? + stopped_service="Worker" + stopped_pid="$worker_pid" + stopped_log="$worker_log" break fi if ! kill -0 "$frontend_pid" 2>/dev/null; then - wait "$frontend_pid" || status=$? + stopped_service="frontend" + stopped_pid="$frontend_pid" + stopped_log="$frontend_log" break fi sleep 1 done +if wait "$stopped_pid"; then + status=0 +else + status=$? +fi + if (( status == 0 )); then - echo "A development service stopped; shutting down the other service." + printf '%s stopped with status 0; shutting down the other services. Log: %s\n' \ + "$stopped_service" "$stopped_log" else - echo "Error: a development service exited with status ${status}." >&2 + printf 'Error: %s exited with status %s. Log: %s\n' \ + "$stopped_service" "$status" "$stopped_log" >&2 fi exit "$status" From afd2eb49481e891d3e97a71d4cc03e5c6bfa9a55 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 01:21:09 +0800 Subject: [PATCH 08/12] docs: close out local recovery maintenance --- docs/NEXT_TASK.md | 2 +- docs/PROJECT_STATUS.md | 8 +++---- docs/phases/PHASE-2-RELIABILITY.md | 2 +- .../2026-08-30-restore-data-quiet-startup.md | 11 +++++---- .../2026-08-30-restore-data-quiet-startup.md | 24 +++++++++++-------- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index d321ba1..ac4822f 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -22,7 +22,7 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 `0006` 兼容修复已完成本地 migration/full test/lint/Mock smoke、真实失败备份副本升级和当前重建库 startup/check；实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 4/4 成功，因此该维护任务为 `completed`，不改变 P2-07 的 `planned` 状态或范围。 -2026-08-30 的个人本地维护又从最新逻辑非空的 SQLite 一致性备份在 staging 中前进迁移并恢复 Demo 数据，同时让 `make dev` 默认把 API/Worker/Vite 详细输出分流到私有 Git 忽略日志。它只处理默认本地开发库和启动体验，不提供 PostgreSQL+keyring backup/restore、Redis rebuild 或告警演练，因而不算 P2-07 功能实现；本文件的下一切片保持不变。 +2026-08-30 的个人本地维护又从最新逻辑非空的 SQLite 一致性备份在 staging 中前进迁移并恢复 Demo 数据，同时让 `make dev` 默认把 API/Worker/Vite 详细输出分流到私有 Git 忽略日志。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 的 [run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 4/4 成功；该维护只处理默认本地开发库和启动体验，不提供 PostgreSQL+keyring backup/restore、Redis rebuild 或告警演练，因而不算 P2-07 功能实现；本文件的下一切片保持不变。 P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index b23a997..e091b64 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -57,18 +57,18 @@ - 真实失败备份副本已无损升级到 `0006` 并通过 integrity/FK/Alembic check；当前重建库由标准 preflight 自动备份后从 canonical `0005` 到 `0006`，startup gate、quick check、FK 和 metadata check 全部通过，业务计数与 Worker facts 保持。 - 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 的 backend、真实 PostgreSQL/Redis、完整 Compose 与 frontend 四个 job 全部成功；本维护任务完成，不改变 Phase 2/P2-07 状态。 -## 2026-08-30 本地数据恢复与静默启动（`in_progress`） +## 2026-08-30 本地数据恢复与静默启动（`completed`） - 默认本地 SQLite 重建库逻辑为空；多份约 96 MiB 候选也因 freelist 较大但业务表全空，不能恢复数据。最新且 revision 最高的非空一致性备份为 `backend/data/llmbenchlab.db.pre-alembic-20260827T073137431634Z.bak`（SHA-256 `7e046c1e7cd4ec39c5fe6f57b34f130670e0d249a70bf052a84a23e085a59a53`），含 1 个 Mock Model、1 个 Demo Benchmark、15 Questions、1 个 completed Run 和 15 Responses；与更早两份非空备份在全部共有列的内容摘要一致。 - 停止 API/Worker/Vite 并确认默认库无占用/sidecar 后，只在该备份的一致性 staging 副本上执行 `0002→0006`。迁移前后五张旧表共有列摘要都为 `d1b3b74b7726f9e7903fbd3f445ad258d5f5aa4b885c976582f2d53e1d30302f`；恢复后的默认库 `quick_check=ok`、外键错误 0、Alembic current=head，API/Web 实际读取到 `1/1/1` 个 Model/Benchmark/Run。原始恢复源未修改；重建空库另存为 `backend/data/llmbenchlab.db.pre-original-data-restore-20260829T170121Z.bak`（SHA-256 `ec2ef8b2d5c9a338ce3e5f94c68a3c5742d288a798df2b7a6096960a48610c90`）。 -- `make dev` 现在只在控制台显示地址和日志位置；API、Worker、Vite 详细输出分别 append 到私有的 Git 忽略日志，单服务 Make 入口仍保留前台诊断。离线启动器 `3 passed`，真实本地启动/live/health/ready/API/Web 探针通过；完整 `make test` 为后端 `930 passed, 33 skipped`、前端 `38 passed`，lint/build/smoke/Compose config 也已通过。提交/push 与精确 SHA CI 尚在收尾，因此当前维护状态仍为 `in_progress`。 +- `make dev` 现在只在控制台显示地址和日志位置；API、Worker、Vite 详细输出分别 append 到私有的 Git 忽略日志，单服务 Make 入口仍保留前台诊断。离线启动器 `3 passed`，真实本地启动/live/health/ready/API/Web 探针通过；完整 `make test` 为后端 `930 passed, 33 skipped`、前端 `38 passed`，lint/build/smoke/Compose config 也已通过。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，[run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 四个必需 job 全部成功，因此本维护为 `completed`。 - 该维护只恢复个人本地 SQLite Demo 数据并改进开发入口，不是 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建或灾难恢复认证；Phase 2/P2-07 状态不变。 ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 - 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 -- 本地数据恢复与静默启动：状态为 `in_progress`；数据恢复和本地验证已完成，等待提交/push 与精确 SHA CI。 +- 本地数据恢复与静默启动：状态为 `completed`；实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，精确 SHA run `33265171953` 4/4 成功。 - P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 @@ -108,7 +108,7 @@ | P2-06 evidence 文档远程 CI | `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 run `33165775037` 4/4 | 精确文档 SHA 门禁完成；P2-06 为 `completed` | | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | -| 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；实现尚待 commit/push 与精确 SHA CI，不改变 P2-07 | +| 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；`5075bdb…` 的 run `33265171953` 4/4 成功，不改变 P2-07 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | | P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | | P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 2ffea0b..79a3179 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -115,7 +115,7 @@ | P2-06 staged 技术/安全终审 | structured-extra High 与 Worker `__main__` logger Medium 已修复；76-file implementation index 为 0 Blocker/High/Medium；hydration/import integrity 目标集 `67 passed` | 已进入 clean implementation commit `9a20676…` | | P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | | 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | `8fb51b6…` 的 exact-SHA run `33263405214` 4/4；不改变 P2-07 planned 状态 | -| 2026-08-30 本地恢复/静默启动维护 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、SQLite digest/quick/FK/head、真实 API/Web 读取通过 | 个人本地 Demo 数据恢复和开发 UX；尚待精确 SHA CI，且不是 P2-07 恢复认证 | +| 2026-08-30 本地恢复/静默启动维护 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、SQLite digest/quick/FK/head、真实 API/Web 读取通过 | 个人本地 Demo 数据恢复和开发 UX；`5075bdb…` 的 run `33265171953` 4/4 成功，且不是 P2-07 恢复认证 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 diff --git a/docs/plans/2026-08-30-restore-data-quiet-startup.md b/docs/plans/2026-08-30-restore-data-quiet-startup.md index 7716afc..60dda89 100644 --- a/docs/plans/2026-08-30-restore-data-quiet-startup.md +++ b/docs/plans/2026-08-30-restore-data-quiet-startup.md @@ -1,7 +1,7 @@ # 恢复本地数据并静默一键启动执行计划 - Owner: Codex -- Status: active +- Status: completed - Created: 2026-08-30 - Updated: 2026-08-30 - Related phase: [Phase 2](../phases/PHASE-2-RELIABILITY.md) @@ -52,7 +52,7 @@ 3. [completed] 实现静默 `make dev` 并增加回归测试。 - Files/modules: `scripts/dev.sh`、`backend/tests/test_dev_script.py` - Validation: 正常启动只输出简洁摘要，三个日志文件接收详细输出，失败状态和清理行为有自动化覆盖。 -4. [in_progress] 运行定向与全量验证，更新强制文档并完成审查、提交、push 与精确 SHA CI。 +4. [completed] 运行定向与全量验证，更新强制文档并完成审查、提交、push 与精确 SHA CI。 - Files/modules: README、部署/运维/安全/测试、CHANGELOG、PROJECT_STATUS、PHASE-2、NEXT_TASK、本计划与工作日志 - Validation: 本地验证通过、diff/秘密扫描通过、远程必需 jobs 对精确 SHA 全绿。 @@ -74,7 +74,7 @@ | 恢复后数据一致 | SQLite count/digest、`alembic current/check` | 业务数据与源一致，revision 为 head | 通过：`1/1/15/1/15`，共有列 digest 一致，quick/FK/head/check 全绿 | | 静默启动行为 | 定向 pytest + 有界 `make dev` 探针 | 控制台简洁、详细日志分流、服务可用 | 通过：`3 passed`；真实启动期间仅两行摘要，API/Web/恢复数据可读，详细日志 `0600` | | 全量质量门禁 | `make test`、`make lint`、frontend build、`make smoke`、Compose config | 全部适用项通过 | 通过：backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config 全绿，build 保留既有 chunk warning | -| 远程门禁 | push 后 GitHub Actions | 精确提交 4/4 必需 job 成功 | 待执行 | +| 远程门禁 | push 后 GitHub Actions | 精确提交 4/4 必需 job 成功 | 通过：`5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 的 [run 33265171953](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 4/4 成功 | ## Rollback @@ -90,9 +90,9 @@ ## Completion evidence - Changed files: `scripts/dev.sh`、`backend/tests/test_dev_script.py`、README、Deployment/Operations/Security/Testing、CHANGELOG、PROJECT_STATUS、Phase 2、NEXT_TASK、本计划与工作日志；另有 Git 忽略的默认 SQLite 恢复和安全备份。 -- Commands run: SQLite 只读审计/backup/staging migration/digest/head 检查；启动器定向 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/Compose config；真实有界 `make dev` 与 API/Web 探针。 +- Commands run: SQLite 只读审计/backup/staging migration/digest/head 检查；启动器定向 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/Compose config；真实有界 `make dev` 与 API/Web 探针；commit/push 和精确 SHA CI 4/4。 - Acceptance evidence: 默认库 `1/1/15/1/15` 且共有列 digest 与源一致；quick/FK/head/check 通过；真实运行控制台仅两行摘要，日志分流为 `0600`，API/Web 可读恢复数据。 -- Not run: 本地真实 PostgreSQL/Redis integration、capacity/SLO/Phase 2 acceptance；本任务不改变其语义，远程必需 jobs 尚待提交后运行。 +- Not run: 本机 capacity/SLO/完整 Phase 2 acceptance；本任务不改变其语义。远程 CI 已运行真实 PostgreSQL/Redis integration 和完整 Compose reliability acceptance 并成功。 - Known issues: 既有 Python 3.14 依赖 warning 和 Vite 662.39 kB chunk warning；2026-08-28 大备份逻辑为空但继续保留。 ## Decision and discovery log @@ -104,3 +104,4 @@ | 2026-08-30 | decision | 选择 `backend/data/llmbenchlab.db.pre-alembic-20260827T073137431634Z.bak`：它是最新、revision 最高且逻辑非空的候选，SHA-256 为 `7e046c1e7cd4ec39c5fe6f57b34f130670e0d249a70bf052a84a23e085a59a53`；与两个更早非空备份在所有共有列上的内容摘要一致。 | 只在该文件的一致性副本上迁移；源文件保持只读不变。 | | 2026-08-30 | discovery | staging 从 `0002` 顺序升级到 `0006` 后，五张原有业务表共有列摘要保持 `d1b3b74b7726f9e7903fbd3f445ad258d5f5aa4b885c976582f2d53e1d30302f`；完整性、外键、head、索引和 active-state 检查通过。 | 允许替换默认库。 | | 2026-08-30 | action | 当前空库先通过 SQLite backup API 保存为 `backend/data/llmbenchlab.db.pre-original-data-restore-20260829T170121Z.bak`，SHA-256 `ec2ef8b2d5c9a338ce3e5f94c68a3c5742d288a798df2b7a6096960a48610c90`；随后用已验证 staging 同目录替换默认库。 | 空库及其 3 条 Worker process facts 可回滚；原恢复源未修改。 | +| 2026-08-30 | validation | 实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push；GitHub Actions run `33265171953` 的 frontend、backend、真实 PostgreSQL/Redis integration 和完整 Compose reliability 四个必需 job 全部成功。 | 远程门禁完成，任务可收尾。 | diff --git a/docs/worklogs/2026-08-30-restore-data-quiet-startup.md b/docs/worklogs/2026-08-30-restore-data-quiet-startup.md index d96333f..6d9bcd8 100644 --- a/docs/worklogs/2026-08-30-restore-data-quiet-startup.md +++ b/docs/worklogs/2026-08-30-restore-data-quiet-startup.md @@ -9,7 +9,7 @@ - 关联阶段：[Phase 2](../phases/PHASE-2-RELIABILITY.md) - 关联计划：[恢复本地数据并静默一键启动](../plans/2026-08-30-restore-data-quiet-startup.md) - 关联 ADR：无；不改变 schema、协议或应用日志合同 -- 最终状态：in_progress +- 最终状态：completed ## 初始仓库状态 @@ -42,7 +42,7 @@ - [x] `make dev` 正常运行时不持续刷 API/Worker/Vite 详细日志，并清楚显示服务地址和日志位置。 - [x] 任一子进程退出仍会停止其余进程；失败时返回非零并指向详细日志。 - [x] 定向/全量测试、lint、build、离线 smoke 和适用部署检查有真实结果，文档与项目状态同步。 -- [ ] 阶段提交 push 到工作分支，对应精确 SHA 的 GitHub Actions 必需 jobs 全部成功。 +- [x] 阶段提交 push 到工作分支，对应精确 SHA 的 GitHub Actions 必需 jobs 全部成功。 ## 假设 @@ -63,7 +63,7 @@ 1. [completed] 只读核验候选备份、当前进程、keyring 边界和启动脚本。 2. [completed] 停止服务、准备/迁移/验真副本并可回滚替换默认库。 3. [completed] 实现静默启动与回归测试。 -4. [in_progress] 运行完整验证、更新文档、审查、提交、push 并等待精确 SHA CI。 +4. [completed] 运行完整验证、更新文档、审查、提交、push 并等待精确 SHA CI。 ## 实际修改 @@ -94,6 +94,7 @@ | 01:05 CST | validation | 首次从 `backend/` 运行定向组合命令时误写 `scripts/dev.sh`，`bash -n`/ShellCheck 因路径不存在失败；该命令未启用 `set -e`，后续 Ruff/pytest 仍通过。随后用 `set -euo pipefail` 和 `../scripts/dev.sh` 完整重跑，全项通过。 | 最终证据只采用纠正后的完整命令，同时保留首次操作偏差。 | | 01:06 CST | validation | 真实 `make dev` 控制台只出现两行摘要；live/health/ready、Web 及 Model/Benchmark/Run 读取通过，重复请求后控制台无新增输出。发送 Ctrl-C 后 Make 报预期中断 130，所有服务/端口退出。 | 用户可正常启动且不再刷屏；详细日志仍可追查。 | | 01:11 CST | review | 独立终审为 0 Blocker/High/Medium；指出 PROJECT_STATUS 仍写“等待完整测试”的 Low 文档残留，已改为只等待提交/push/CI。可信本地日志目录的同用户 symlink 边界为非阻断 Low，默认目录 `0700` 且威胁模型不承诺防同权限攻击者，本次不扩成安全文件安装器。 | 关闭文档不一致，保持最小范围。 | +| 01:19 CST | validation | 实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已普通 push；GitHub Actions run `33265171953` 的 frontend、backend、真实 PostgreSQL/Redis integration 和完整 Compose reliability 四个必需 job 全部成功。 | 精确实现 SHA 远程门禁完成。 | ## 实际运行命令 @@ -118,6 +119,8 @@ | `docker compose config --quiet` | 部署配置语法 | 0 | 通过，未启动容器 | | `alembic current && alembic check` + 默认库 quick/FK/count | 恢复后最终 schema/data 复核 | 0 | `0006 (head)`、无新操作、quick ok、FK 0、`1/1/15/1/15` | | 真实启动后的共有列 digest/Worker process 核验 | 确认启动探针未改历史业务数据 | 0 | digest 仍为 `d1b3b74b…0302f`；新增 1 条正常 stopped Worker process，active 为 0 | +| `git commit` + `git push origin codex/complete-evaluation-workflow` | 提交并推送实现阶段 | 0 | commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已推送 | +| `gh run watch 33265171953 --exit-status` | 等待精确实现 SHA 远程门禁 | 0 | 四个必需 job 全部 success；仅现有 GitHub Actions Node runtime deprecation 注解 | ## 测试结果 @@ -125,14 +128,15 @@ - 失败：最终验证 0；曾有一次只读 reservation 列名错误和一次检查路径错误，均已记录并完整纠正重跑 - Lint/typecheck/build：Ruff/format 154 files、ESLint、TypeScript、Vite build 全部通过；保留既有 662.39 kB chunk warning - Smoke/Docker：离线 Mock smoke `1 passed, 7 deselected`；Compose config 通过且未启动容器 +- 远程 CI：frontend、backend、真实 PostgreSQL/Redis integration、完整 Compose reliability 4/4 success ## 未运行验证 -- 真实 PostgreSQL/Redis integration、capacity、SLO 和 Phase 2 acceptance：未运行；本次只修改本地启动器和个人 SQLite 数据，不改变其生产/并发语义，远程 CI 仍会执行仓库必需 integration/full-stack jobs。 +- 本机 capacity、SLO 和完整 Phase 2 acceptance：未运行；本次不改变其语义。远程 CI 已执行真实 PostgreSQL/Redis integration 与完整 Compose reliability acceptance 并成功。 ## 未完成项 -- 数据库恢复、代码、文档和本地全量验证已完成；仍需终审、提交/push 与精确 SHA CI。 +- 无；约定范围、实现/文档、本地验证、实现提交/push 与精确 SHA CI 均已完成。 ## 已知问题与限制 @@ -140,20 +144,20 @@ ## 安全检查 -- 真实密钥扫描：未暂存 diff 的高置信模式扫描和 `git diff --check` 已通过；提交前仍复核 staged diff；核验未读取或输出 keyring 内容、credential envelope 或 Provider Key +- 真实密钥扫描：未暂存与 staged diff 的高置信模式扫描、`git diff --check` 均通过；核验未读取或输出 keyring 内容、credential envelope 或 Provider Key - 真实 API 调用：否 - 日志/API 脱敏：应用日志合同不变；组合启动详细输出保留到 Git 忽略目录，目录/文件收紧为 `0700/0600` - 危险 Git 操作（force push/reset 等）：无 -- 阶段 push：待执行 -- 远程 CI：待执行 +- 阶段 push：`origin/codex/complete-evaluation-workflow`，实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b`，成功 +- 远程 CI：精确实现 SHA 的 [run 33265171953](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 4/4 必需 job 成功 - 遗留安全风险：数据库与 keyring 必须配对恢复；若存在 stored credential 且 keyring 不匹配，应保持 fail closed ## 结果与下一步 -进行中；数据、实现、文档和本地门禁已经完成，下一步是终审、staged diff/秘密复核、commit/push 和精确 SHA CI。 +任务完成：默认本地 SQLite 已恢复原 Demo 数据，组合启动控制台保持简洁且详细日志可追查，全部本地和远程门禁通过。[NEXT_TASK.md](../NEXT_TASK.md) 仍指向原定 P2-07 最小 verifier，本维护不改变路线图。 ## 最终 Git 状态 ```text -任务进行中 +## codex/complete-evaluation-workflow...origin/codex/complete-evaluation-workflow ``` From 0163b67c00eb59ae59db5f3adb679ad85c799142 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 01:38:08 +0800 Subject: [PATCH 09/12] docs: record local benchmark imports --- CHANGELOG.md | 1 + docs/NEXT_TASK.md | 2 + docs/PROJECT_STATUS.md | 12 ++- docs/phases/PHASE-3-BENCHMARKS.md | 5 + .../2026-08-30-load-downloaded-benchmarks.md | 97 +++++++++++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 docs/worklogs/2026-08-30-load-downloaded-benchmarks.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 802001e..4584f43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Changed +- Loaded the three already prepared, Git-ignored standard dataset ZIPs into the default personal SQLite through the existing validated import API: GPQA-Diamond (198 questions), MMLU-Pro Direct (12,032), and MMLU-Pro Official-CoT (12,032). This is a local data-maintenance result, not redistributed third-party data or a product/protocol change; the pre-import database backup remains local and no Provider was called. - Changed the combined `make dev` launcher to keep the console quiet after a concise address/log summary. API, Worker, and Vite output is appended to separate Git-ignored `artifacts/dev-logs` files with UTC session markers and private local permissions; individual service Make targets remain foreground diagnostics, and child failures still propagate their status. - Governed production logging at the source boundary: application calls use literal argument-free messages, structured values remain allowlisted and finite, third-party logger text/identity is replaced by fixed classification, and the raw Uvicorn access handler is disabled rather than bypassing the JSON redaction contract. - Extended the Phase 2 migration/importer head from `20260827_0004` / 12 tables to `20260828_0005` / 13 tables. A populated `worker_processes` table blocks `0005 -> 0004` before DDL; only an explicitly emptied table may downgrade, after which the existing governance/audit `0004` guards still apply. diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index ac4822f..1354a5d 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -24,6 +24,8 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 2026-08-30 的个人本地维护又从最新逻辑非空的 SQLite 一致性备份在 staging 中前进迁移并恢复 Demo 数据，同时让 `make dev` 默认把 API/Worker/Vite 详细输出分流到私有 Git 忽略日志。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 的 [run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 4/4 成功；该维护只处理默认本地开发库和启动体验，不提供 PostgreSQL+keyring backup/restore、Redis rebuild 或告警演练，因而不算 P2-07 功能实现；本文件的下一切片保持不变。 +同日又把 `artifacts/benchmarks/` 中已存在的 GPQA-Diamond 与两种 MMLU-Pro profile 通过正式导入 API 加载到默认个人 SQLite，当前为 `4` 个 Benchmark、`24,277` 题，原 Model/Run/Response 保持且没有 Provider 调用。该本地数据操作不提交第三方题目、不改变产品实现或路线图，P2-07 的下一任务仍保持不变。 + P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 ## 已完成：建立 P2-07 工作包 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index e091b64..0e1bdba 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -64,11 +64,19 @@ - `make dev` 现在只在控制台显示地址和日志位置；API、Worker、Vite 详细输出分别 append 到私有的 Git 忽略日志，单服务 Make 入口仍保留前台诊断。离线启动器 `3 passed`，真实本地启动/live/health/ready/API/Web 探针通过；完整 `make test` 为后端 `930 passed, 33 skipped`、前端 `38 passed`，lint/build/smoke/Compose config 也已通过。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，[run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 四个必需 job 全部成功，因此本维护为 `completed`。 - 该维护只恢复个人本地 SQLite Demo 数据并改进开发入口，不是 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建或灾难恢复认证；Phase 2/P2-07 状态不变。 +## 2026-08-30 已下载标准评测集本地加载（`in_progress`） + +- `artifacts/benchmarks/` 中三个 Git 忽略 dataset-v1 ZIP 均由当前 Loader 校验通过：GPQA-Diamond `198` 题、MMLU-Pro Direct `12,032` 题、MMLU-Pro Official-CoT `12,032` 题。原始 `dataset-cache` 仍只作为固定来源缓存，没有被当作导入包。 +- 在现有 `make dev` 服务无活动 Run 时，先用 SQLite 在线 backup 创建 `0600` 的 `backend/data/llmbenchlab.db.pre-benchmark-import-20260829T173032Z.bak`（SHA-256 `d8c71c44b0ee364030d0053788f34bd985a443ba30a9cd17f1069a9737d10206`），再通过正式 `/api/v1/benchmarks/import` 逐个导入，三个请求均为 `201`。 +- 默认个人 SQLite 现有 `4` 个 Benchmark、`24,277` 道题；逐集持久化题数与 manifest 一致。原有 `1` 个 Model、`1` 个 completed Run 和 `15` 条 Response 不变，active Run 为 `0`；`quick_check=ok`、外键错误 `0`、Alembic head=`20260829_0006`，API 列表/逐集 total 也已对账。 +- 目标 Loader/标准转换器离线测试 `40 passed`；未调用真实 Provider，未修改 Schema/API/协议或产品代码。当前只等待仓库文档提交、push 与精确 SHA CI，不改变 Phase 3/P2-07 状态。 + ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 - 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 - 本地数据恢复与静默启动：状态为 `completed`；实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，精确 SHA run `33265171953` 4/4 成功。 +- 已下载标准评测集本地加载：三个现有正式 ZIP 已导入并完成本地数据库/API/目标测试验证；仓库文档门禁仍为 `in_progress`。 - P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 @@ -109,6 +117,7 @@ | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；`5075bdb…` 的 run `33265171953` 4/4 成功，不改变 P2-07 | +| 2026-08-30 标准评测集本地加载 | 三个 ZIP Loader 校验通过；API 导入 `201/201/201`；数据库/API 对账为 `4` Benchmarks、`24,277` Questions；`quick_check=ok`、FK `0`、head `0006`；目标测试 `40 passed` | 本地加载完成，原 Model/Run/Response 保持；文档 commit/push/精确 SHA CI 待执行，无 Provider 调用 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | | P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | | P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | @@ -133,7 +142,8 @@ - [Phase 2 正式 SLO 与容量模型](worklogs/2026-08-28-phase-2-slo-capacity-model.md) - [Phase 2 可观测性与审计保留](worklogs/2026-08-28-phase-2-observability-retention.md) - [本地数据恢复与静默启动](worklogs/2026-08-30-restore-data-quiet-startup.md) +- [加载已下载的标准评测集](worklogs/2026-08-30-load-downloaded-benchmarks.md) ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。本次个人本地 SQLite/启动器维护不改变路线图：P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。本次个人本地 SQLite/启动器维护与标准评测集加载都不改变路线图：P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 与 Phase 3 继续保持 `in_progress`。 diff --git a/docs/phases/PHASE-3-BENCHMARKS.md b/docs/phases/PHASE-3-BENCHMARKS.md index b842657..3f1c7ae 100644 --- a/docs/phases/PHASE-3-BENCHMARKS.md +++ b/docs/phases/PHASE-3-BENCHMARKS.md @@ -52,6 +52,11 @@ MMLU-Pro/GPQA 的可信本地切片。该决定不满足代码沙箱、全局预 沙箱范围未开始。该 UI 切片的完整本地门禁已通过，功能提交 `467d0243b4fb081c2d637b20ee0958c3bd6ee6d1` 已 push； 分支无 PR且精确 SHA 未触发仅监听 PR/main 的 workflow，不能称为远程绿色。 +2026-08-30 的个人本地实例已把 `artifacts/benchmarks/` 中现有的三份可复现 ZIP 经正式 API 加载： +GPQA-Diamond `198` 题，以及 MMLU-Pro Direct/Official-CoT 各 `12,032` 题；默认库连同 Demo 共 +`4` 个 Benchmark、`24,277` 题。第三方题目和导入前备份继续位于 Git 忽略目录，本次没有重新下载、 +运行评测或调用 Provider，也不代表 IFEval、Plugin SDK、沙箱、分组 UI 或本阶段验收已经完成。 + ## 验收标准 - [ ] 每个标准 Benchmark 可从固定来源/版本重复导入并得到相同 Hash。 diff --git a/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md b/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md new file mode 100644 index 0000000..acfeb74 --- /dev/null +++ b/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md @@ -0,0 +1,97 @@ +# 2026-08-30：加载已下载的标准评测集 + +## 元数据 + +- 日期：2026-08-30（Asia/Shanghai） +- 分支：`codex/complete-evaluation-workflow` +- 当前阶段：Phase 3 标准 Benchmark 的既有可信本地切片 +- 关联计划：不适用；这是一次单一、可回滚的本地数据导入，不改变架构、协议、Schema 或公共 API +- 初始工作区：与 `origin/codex/complete-evaluation-workflow` 同步，开始时无未提交改动 + +## 目标与背景 + +用户已经在仓库的 Git 忽略目录中下载并转换了标准评测集，希望把这些现有评测集加载到默认本地数据库，而不是重新下载。当前默认 SQLite 仅包含恢复后的 Demo Benchmark。 + +## 范围 + +- 只读盘点并校验 `artifacts/benchmarks/` 中现有的 dataset-v1 ZIP。 +- 通过项目已有的 Benchmark 导入入口，将全部唯一且有效的 ZIP 导入默认本地 SQLite。 +- 导入前创建一致性备份；导入后验证数据库、API 与逐 Benchmark 题数。 +- 记录本地数据变更、验证证据和回滚位置。 + +## 非目标 + +- 不重新下载或转换第三方数据。 +- 不运行任何评测，不调用真实 Provider，不产生模型费用。 +- 不修改数据格式、协议、数据库 Schema、API 或产品代码。 +- 不实现 IFEval、插件 SDK、代码沙箱或 Phase 3 其余 Roadmap 内容。 + +## 验收标准 + +- 三个现有 ZIP 都通过当前 Loader 校验，且不输出题目正文。 +- `gpqa-diamond`、`mmlu-pro-direct`、`mmlu-pro-official-cot` 均可由默认 API 查询。 +- 导入后共 4 个 Benchmark、24,277 道题；原有 1 个 Model、1 个 completed Run 和 15 条 Response 不变。 +- SQLite `quick_check` 为 `ok`、外键错误为 0、Alembic revision 为当前 head。 +- 导入前备份保留，未触发 Provider 请求，未留下新增活动 Run。 + +## 假设 + +- 用户所说“目录里下载的评测集”指 `artifacts/benchmarks/` 中三个已生成的 dataset-v1 ZIP；`artifacts/dataset-cache/` 是转换器源缓存，不是可直接导入包。 +- 两份 MMLU-Pro ZIP 虽来自相同题目源，但 prompt/profile 不同，应作为两个独立 Benchmark 保留。 + +## 风险与控制 + +- 大批量写入中断：导入前使用 SQLite 在线 backup API 创建唯一备份；每个 ZIP 由 API 单独事务提交。 +- 重复或冲突版本：导入前比较 slug/version/hash；现有持久化语义对同 hash 幂等、异 hash 冲突拒绝。 +- 第三方题目敏感性：只记录文件名、大小、Hash 和计数，不打印或提交题目正文；数据与备份继续留在 Git 忽略目录。 +- 运行中服务并发：只使用公开导入 API，不直接改写被服务占用的数据库；导入后重新核对原有 Run/Response 计数。 + +## 实施步骤 + +1. 完成仓库、阶段、格式/API/安全/测试文档和工作区只读检查。 +2. 校验三个 ZIP 的 archive SHA-256、dataset hash、slug/version 和题数，并核对数据库不存在同版本。 +3. 创建并验证默认 SQLite 的导入前备份。 +4. 通过现有 `/api/v1/benchmarks/import` 顺序导入三个 ZIP。 +5. 验证 API 列表、逐集题数、数据库完整性、迁移 head 及原有业务计数。 +6. 运行相关离线测试，更新状态文档，复核 diff 和秘密边界。 +7. 独立提交并普通 push，等待精确 SHA 的必需 GitHub Actions 全绿。 + +## 实际结果 + +### 盘点与导入 + +| ZIP | archive SHA-256 | dataset hash | 题数 | API | +| --- | --- | --- | ---: | --- | +| `gpqa-diamond-1.0.0-dc4dc6fd5089.zip` | `68623d1840f14ad6c18163c5e87e47e78943961c5f8bf5d70fb9258ba074b62c` | `38e196d2993f7d1f59ad077c93ac3c128ad8765264c897bae9a3b324f6caae53` | 198 | `201` | +| `mmlu-pro-direct-1.0.0-44a485f6edcb.zip` | `e62a1f45cba71d18f4096256b08bbd4b182d1dd9e1a4d81e6de007db715c27a6` | `cf7872819f2be58373e8181706aa366b23008ac664f3fc551aa82145a758500b` | 12,032 | `201` | +| `mmlu-pro-official-cot-1.0.0-8b7fc4ca4a06.zip` | `f7bc8773a3fa4937cd482cc0bf65a42ec818b3d386320cb23299114acde329bf` | `81e348f25f217baab178df9217909438da1a00707473f683cac087152911e33c` | 12,032 | `201` | + +- 三个 ZIP 都是 `0600` 普通文件，并由当前 `load_dataset_zip_bytes` 完整校验；没有输出题目正文。 +- 导入时现有 `make dev` 正常运行，Worker 扫描但无 `pending/running` Run。按正式入口顺序执行三个 multipart 请求，没有直接写被服务占用的 SQLite。 +- 导入前备份：`backend/data/llmbenchlab.db.pre-benchmark-import-20260829T173032Z.bak`，SHA-256 `d8c71c44b0ee364030d0053788f34bd985a443ba30a9cd17f1069a9737d10206`，权限 `0600`；备份自身 `quick_check=ok`、外键错误 `0`、head=`20260829_0006`，业务计数为 `1/1/15/1/15`（Model/Benchmark/Question/Run/Response）。 + +### 导入后验证 + +- 默认数据库：`quick_check=ok`、外键错误 `0`、Alembic `20260829_0006`。 +- 总计：`1` Model、`4` Benchmarks、`24,277` Questions、`1` completed Run、`15` Responses、`0` active Runs。 +- DB 逐集 `COUNT(questions)` 与 manifest/API 均一致：Demo `15`、GPQA `198`、MMLU-Pro Direct `12,032`、MMLU-Pro Official-CoT `12,032`。 +- API `/benchmarks` 返回 total `4` 且三个 slug/version/dataset hash 精确匹配；三个 `/questions?limit=1` 响应 total 分别为 `198/12032/12032`，只投影计数用于记录。 +- API、Worker 与 Vite 保持用户启动时的运行状态；没有创建 Run、访问 Provider 或产生费用。 + +### 实际命令与结果 + +| 命令/检查 | 结果 | +| --- | --- | +| Loader 盘点三个 `artifacts/benchmarks/*.zip` | 3/3 合法；原始 cache 未作为导入包 | +| `sqlite3 ... .backup ...pre-benchmark-import...bak` | 成功；备份校验与业务计数通过 | +| 三次 `curl -F archive=@... /api/v1/benchmarks/import` | `201/201/201` | +| SQLite quick/FK/head/count/逐集对账 | 通过；`4` Benchmarks / `24,277` Questions | +| API Benchmark 列表与逐集 total 对账 | 通过 | +| `uv run pytest -q tests/test_dataset_loader.py tests/test_standard_datasets.py` | `40 passed`；只有既有上游弃用 warning | + +### 已知边界与未运行项 + +- 本次没有产品代码、Schema、API、协议或前端行为变更，因此本地未重复全量 `make test`、lint、frontend build、Smoke 或 Compose；目标 Loader/转换器测试和真实本地导入/数据库/API 对账是本次最小充分验证，远程 CI 将执行仓库常规门禁。 +- 两种 MMLU-Pro profile 共享源题但 prompt 配置不同，结果不得互相或与其他 profile/hash 无提示混合比较。 +- 本地 SQLite 与第三方题目/备份不进入 Git；备份 hash 是完整性标识，不是签名、加密或访问控制。 +- 当前状态：本地数据加载完成；文档提交、普通 push 与精确 SHA GitHub Actions 待执行，所以任务暂保持 `in_progress`。 From a2369d78fc3b1f2674c7882bdb7541c168861651 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 01:43:50 +0800 Subject: [PATCH 10/12] docs: close local benchmark import --- CHANGELOG.md | 2 +- docs/NEXT_TASK.md | 2 +- docs/PROJECT_STATUS.md | 8 ++++---- docs/phases/PHASE-3-BENCHMARKS.md | 3 ++- docs/worklogs/2026-08-30-load-downloaded-benchmarks.md | 8 ++++++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4584f43..8f910a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Changed -- Loaded the three already prepared, Git-ignored standard dataset ZIPs into the default personal SQLite through the existing validated import API: GPQA-Diamond (198 questions), MMLU-Pro Direct (12,032), and MMLU-Pro Official-CoT (12,032). This is a local data-maintenance result, not redistributed third-party data or a product/protocol change; the pre-import database backup remains local and no Provider was called. +- Loaded the three already prepared, Git-ignored standard dataset ZIPs into the default personal SQLite through the existing validated import API: GPQA-Diamond (198 questions), MMLU-Pro Direct (12,032), and MMLU-Pro Official-CoT (12,032). This is a local data-maintenance result, not redistributed third-party data or a product/protocol change; the pre-import database backup remains local and no Provider was called. Documentation commit `0163b67c00eb59ae59db5f3adb679ad85c799142` was pushed, and its exact-SHA GitHub Actions run `33266167547` passed all four required jobs. - Changed the combined `make dev` launcher to keep the console quiet after a concise address/log summary. API, Worker, and Vite output is appended to separate Git-ignored `artifacts/dev-logs` files with UTC session markers and private local permissions; individual service Make targets remain foreground diagnostics, and child failures still propagate their status. - Governed production logging at the source boundary: application calls use literal argument-free messages, structured values remain allowlisted and finite, third-party logger text/identity is replaced by fixed classification, and the raw Uvicorn access handler is disabled rather than bypassing the JSON redaction contract. - Extended the Phase 2 migration/importer head from `20260827_0004` / 12 tables to `20260828_0005` / 13 tables. A populated `worker_processes` table blocks `0005 -> 0004` before DDL; only an explicitly emptied table may downgrade, after which the existing governance/audit `0004` guards still apply. diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 1354a5d..99da59c 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -24,7 +24,7 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 2026-08-30 的个人本地维护又从最新逻辑非空的 SQLite 一致性备份在 staging 中前进迁移并恢复 Demo 数据，同时让 `make dev` 默认把 API/Worker/Vite 详细输出分流到私有 Git 忽略日志。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 的 [run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 4/4 成功；该维护只处理默认本地开发库和启动体验，不提供 PostgreSQL+keyring backup/restore、Redis rebuild 或告警演练，因而不算 P2-07 功能实现；本文件的下一切片保持不变。 -同日又把 `artifacts/benchmarks/` 中已存在的 GPQA-Diamond 与两种 MMLU-Pro profile 通过正式导入 API 加载到默认个人 SQLite，当前为 `4` 个 Benchmark、`24,277` 题，原 Model/Run/Response 保持且没有 Provider 调用。该本地数据操作不提交第三方题目、不改变产品实现或路线图，P2-07 的下一任务仍保持不变。 +同日又把 `artifacts/benchmarks/` 中已存在的 GPQA-Diamond 与两种 MMLU-Pro profile 通过正式导入 API 加载到默认个人 SQLite，当前为 `4` 个 Benchmark、`24,277` 题，原 Model/Run/Response 保持且没有 Provider 调用。记录 commit `0163b67c00eb59ae59db5f3adb679ad85c799142` 已 push，其精确 SHA run `33266167547` 4/4 成功；该本地数据操作不提交第三方题目、不改变产品实现或路线图，P2-07 的下一任务仍保持不变。 P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 0e1bdba..10247d3 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -64,19 +64,19 @@ - `make dev` 现在只在控制台显示地址和日志位置；API、Worker、Vite 详细输出分别 append 到私有的 Git 忽略日志，单服务 Make 入口仍保留前台诊断。离线启动器 `3 passed`，真实本地启动/live/health/ready/API/Web 探针通过；完整 `make test` 为后端 `930 passed, 33 skipped`、前端 `38 passed`，lint/build/smoke/Compose config 也已通过。实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，[run `33265171953`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33265171953) 四个必需 job 全部成功，因此本维护为 `completed`。 - 该维护只恢复个人本地 SQLite Demo 数据并改进开发入口，不是 P2-07 的 PostgreSQL+keyring backup/restore、Redis 重建或灾难恢复认证；Phase 2/P2-07 状态不变。 -## 2026-08-30 已下载标准评测集本地加载（`in_progress`） +## 2026-08-30 已下载标准评测集本地加载（`completed`） - `artifacts/benchmarks/` 中三个 Git 忽略 dataset-v1 ZIP 均由当前 Loader 校验通过：GPQA-Diamond `198` 题、MMLU-Pro Direct `12,032` 题、MMLU-Pro Official-CoT `12,032` 题。原始 `dataset-cache` 仍只作为固定来源缓存，没有被当作导入包。 - 在现有 `make dev` 服务无活动 Run 时，先用 SQLite 在线 backup 创建 `0600` 的 `backend/data/llmbenchlab.db.pre-benchmark-import-20260829T173032Z.bak`（SHA-256 `d8c71c44b0ee364030d0053788f34bd985a443ba30a9cd17f1069a9737d10206`），再通过正式 `/api/v1/benchmarks/import` 逐个导入，三个请求均为 `201`。 - 默认个人 SQLite 现有 `4` 个 Benchmark、`24,277` 道题；逐集持久化题数与 manifest 一致。原有 `1` 个 Model、`1` 个 completed Run 和 `15` 条 Response 不变，active Run 为 `0`；`quick_check=ok`、外键错误 `0`、Alembic head=`20260829_0006`，API 列表/逐集 total 也已对账。 -- 目标 Loader/标准转换器离线测试 `40 passed`；未调用真实 Provider，未修改 Schema/API/协议或产品代码。当前只等待仓库文档提交、push 与精确 SHA CI，不改变 Phase 3/P2-07 状态。 +- 目标 Loader/标准转换器离线测试 `40 passed`；未调用真实 Provider，未修改 Schema/API/协议或产品代码。仓库记录 commit [`0163b67c00eb59ae59db5f3adb679ad85c799142`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/0163b67c00eb59ae59db5f3adb679ad85c799142) 已 push，其精确 SHA 的 [run `33266167547`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33266167547) 四个必需 job 全部成功；本地加载维护为 `completed`，不改变 Phase 3/P2-07 状态。 ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 - 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 - 本地数据恢复与静默启动：状态为 `completed`；实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，精确 SHA run `33265171953` 4/4 成功。 -- 已下载标准评测集本地加载：三个现有正式 ZIP 已导入并完成本地数据库/API/目标测试验证；仓库文档门禁仍为 `in_progress`。 +- 已下载标准评测集本地加载：状态为 `completed`；三个现有正式 ZIP 已导入并完成本地数据库/API/目标测试验证，`0163b67…` 的 run `33266167547` 4/4 成功。 - P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 @@ -117,7 +117,7 @@ | P2-01 实现远程 CI | `b6a35fe…` run `33146681285` 4/4 | 精确实现 SHA 全绿；PR #2 已于 2026-08-28 合并 | | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；`5075bdb…` 的 run `33265171953` 4/4 成功，不改变 P2-07 | -| 2026-08-30 标准评测集本地加载 | 三个 ZIP Loader 校验通过；API 导入 `201/201/201`；数据库/API 对账为 `4` Benchmarks、`24,277` Questions；`quick_check=ok`、FK `0`、head `0006`；目标测试 `40 passed` | 本地加载完成，原 Model/Run/Response 保持；文档 commit/push/精确 SHA CI 待执行，无 Provider 调用 | +| 2026-08-30 标准评测集本地加载 | 三个 ZIP Loader 校验通过；API 导入 `201/201/201`；数据库/API 对账为 `4` Benchmarks、`24,277` Questions；`quick_check=ok`、FK `0`、head `0006`；目标测试 `40 passed` | 本地加载完成，原 Model/Run/Response 保持；`0163b67…` 的 run `33266167547` 4/4 成功，无 Provider 调用 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | | P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | | P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | diff --git a/docs/phases/PHASE-3-BENCHMARKS.md b/docs/phases/PHASE-3-BENCHMARKS.md index 3f1c7ae..03f0e6f 100644 --- a/docs/phases/PHASE-3-BENCHMARKS.md +++ b/docs/phases/PHASE-3-BENCHMARKS.md @@ -55,7 +55,8 @@ MMLU-Pro/GPQA 的可信本地切片。该决定不满足代码沙箱、全局预 2026-08-30 的个人本地实例已把 `artifacts/benchmarks/` 中现有的三份可复现 ZIP 经正式 API 加载： GPQA-Diamond `198` 题，以及 MMLU-Pro Direct/Official-CoT 各 `12,032` 题；默认库连同 Demo 共 `4` 个 Benchmark、`24,277` 题。第三方题目和导入前备份继续位于 Git 忽略目录，本次没有重新下载、 -运行评测或调用 Provider，也不代表 IFEval、Plugin SDK、沙箱、分组 UI 或本阶段验收已经完成。 +运行评测或调用 Provider；记录 commit `0163b67c00eb59ae59db5f3adb679ad85c799142` 的精确 SHA CI +run `33266167547` 已 4/4 成功。这仍不代表 IFEval、Plugin SDK、沙箱、分组 UI 或本阶段验收已经完成。 ## 验收标准 diff --git a/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md b/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md index acfeb74..d6649db 100644 --- a/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md +++ b/docs/worklogs/2026-08-30-load-downloaded-benchmarks.md @@ -88,10 +88,14 @@ | SQLite quick/FK/head/count/逐集对账 | 通过；`4` Benchmarks / `24,277` Questions | | API Benchmark 列表与逐集 total 对账 | 通过 | | `uv run pytest -q tests/test_dataset_loader.py tests/test_standard_datasets.py` | `40 passed`；只有既有上游弃用 warning | +| `git diff --cached --check`、高置信秘密扫描、独立只读终审 | 通过；0 Blocker/High/Medium/Low，未发现题目正文或秘密 | +| `git commit` / `git push origin codex/complete-evaluation-workflow` | 记录 commit `0163b67c00eb59ae59db5f3adb679ad85c799142` 已普通 push | +| `gh run watch 33266167547 --exit-status` | backend、真实 PostgreSQL/Redis integration、真实 Compose acceptance、frontend 4/4 成功 | ### 已知边界与未运行项 -- 本次没有产品代码、Schema、API、协议或前端行为变更，因此本地未重复全量 `make test`、lint、frontend build、Smoke 或 Compose；目标 Loader/转换器测试和真实本地导入/数据库/API 对账是本次最小充分验证，远程 CI 将执行仓库常规门禁。 +- 本次没有产品代码、Schema、API、协议或前端行为变更，因此本地未重复全量 `make test`、lint、frontend build、Smoke 或 Compose；目标 Loader/转换器测试和真实本地导入/数据库/API 对账是本次最小充分验证，远程 CI 已执行并通过仓库常规门禁。 - 两种 MMLU-Pro profile 共享源题但 prompt 配置不同，结果不得互相或与其他 profile/hash 无提示混合比较。 - 本地 SQLite 与第三方题目/备份不进入 Git；备份 hash 是完整性标识，不是签名、加密或访问控制。 -- 当前状态：本地数据加载完成；文档提交、普通 push 与精确 SHA GitHub Actions 待执行，所以任务暂保持 `in_progress`。 +- 初次 `gh pr status --json currentBranch` 因本机 `gh` 不支持该 JSON 字段而失败；随后改用 `gh pr view 3 --json headRefOid,statusCheckRollup`，精确确认 PR head 与四个 check，不影响产品或验证结果。 +- 状态：本地数据加载、仓库记录提交、普通 push 与精确 SHA GitHub Actions 均完成；任务为 `completed`，Phase 3/P2-07 状态保持不变。 From c6212db2376ffc6b5f32c46b000aad8e7faf9b1f Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 03:10:01 +0800 Subject: [PATCH 11/12] fix: separate observational token usage from hard reservations --- CHANGELOG.md | 1 + README.md | 7 +- ...0830_0007_repair_observational_overdraw.py | 145 +++++++ backend/app/db/import_sqlite.py | 30 +- backend/app/db/prepare_migrations.py | 24 +- backend/app/governance/audit_archive.py | 1 + backend/app/governance/repository.py | 91 +++-- backend/app/runners/evaluation_runner.py | 18 +- .../tests/integration/test_postgres_leases.py | 2 - backend/tests/test_audit_archive.py | 7 +- .../test_evaluation_runner_reliability.py | 2 +- backend/tests/test_governance.py | 195 +++++++++- backend/tests/test_migrations.py | 357 +++++++++++++++++- .../tests/test_phase2_acceptance_script.py | 2 +- backend/tests/test_phase2_capacity_script.py | 4 +- backend/tests/test_sqlite_postgres_import.py | 32 ++ docs/API.md | 8 +- docs/ARCHITECTURE.md | 6 +- docs/DEPLOYMENT.md | 5 +- docs/NEXT_TASK.md | 14 +- docs/OPERATIONS.md | 26 +- docs/PROJECT_STATUS.md | 25 +- docs/ROADMAP.md | 9 +- docs/SECURITY.md | 6 +- docs/TESTING.md | 8 +- ...ability-worker-progress-audit-retention.md | 6 +- ...esql-keyring-recovery-and-redis-rebuild.md | 6 +- ...hema-equivalent-governance-index-repair.md | 2 + ...ken-estimates-are-not-hard-reservations.md | 50 +++ docs/phases/PHASE-2-RELIABILITY.md | 14 +- ...-08-30-fix-observational-token-overdraw.md | 73 ++++ ...-08-30-fix-observational-token-overdraw.md | 90 +++++ frontend/src/pages/RunDetailPage.tsx | 2 +- frontend/tests/run-detail-page.test.tsx | 15 + scripts/phase2_acceptance.py | 3 +- scripts/phase2_capacity.py | 9 +- 36 files changed, 1165 insertions(+), 130 deletions(-) create mode 100644 backend/alembic/versions/20260830_0007_repair_observational_overdraw.py create mode 100644 docs/decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md create mode 100644 docs/plans/2026-08-30-fix-observational-token-overdraw.md create mode 100644 docs/worklogs/2026-08-30-fix-observational-token-overdraw.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f910a2..1305fbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Fixed +- Fixed managed Runs without an explicit `input_token_reservation` being permanently exhausted when Provider actual input exceeded a UTF-8 observational estimate. New attempts now leave input reservation and reserved cost unset in that case while preserving Provider actual usage; explicit input/output reservations and reserved cost derived from complete bounds plus frozen prices still enforce overdraw. Added data-only Alembic head `20260830_0007` to recompute only `governance_scopes.overdrawn`, preserve historical ledger/actual/Response/Run facts, reject upgrade or downgrade while any reservation is active, and restore the old derived predicate on downgrade. Run Detail now uses neutral historical wording for overdraw instead of attributing every case to conservative settlement. - Added ADR-0017 and the forward-only, schema-equivalent Alembic repair revision `20260829_0006` for databases that executed an early `0004` variant before three governance indexes were present. Migration preflight accepts canonical schemas or only a missing subset of those three indexes, so an interrupted SQLite repair is resumable; it backs up SQLite, validates newly historical PostgreSQL `0005` metadata, rejects multiple active policies, and continues to reject every other schema drift. - Prevented external `LogRecord` extras from injecting allowlisted request/run fields, rejected FIFO/non-regular archive inputs without blocking by opening them nonblocking before `fstat`, and made oversized audit archives fail on their global line cap before decoding attacker-controlled lines. - Made importer post-commit verification and no-op retention post-verification fail closed instead of treating an unchanged row count as sufficient evidence; PostgreSQL retention paths now preserve their advisory/row-lock gate through the mutation decision. diff --git a/README.md b/README.md index 8480ea6..6457c2a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - Phase 0（治理、需求、架构和协议）已完成。 - Phase 1 MVP 已具备完整垂直链路：注册模型、载入/导入 Benchmark、创建 Run、逐题持久化、结果聚合和前端展示。 - Phase 2 候选已通过真实 PostgreSQL/Redis 和进程故障验证：除租约、心跳、fencing、幂等 Response 与数据库恢复外，Web/API managed Run 还具有 global/provider/model/run 四层数据库 admission、fixed-minute RPM/TPM、lifetime request/Token/cost budget、有限 backlog、公平 slice、逐 attempt reservation/settlement、typed audit 和历史延迟。 +- 当前 Alembic head 为 data-only `20260830_0007`：没有显式 `input_token_reservation` 时，输入 Token 估算只用于观测，不再写成 hard reservation 或参与 cost/overdraw 裁决；Provider actual usage 仍完整保存。显式 input/output 预留及由完整上界和价格计算的 reserved cost 超额仍按原规则 fail closed。 - 可信本地 CLI 已提供 MMLU-Pro 与 GPQA-Diamond 的固定来源转换、真实 OpenAI-compatible 预检、可恢复执行和完整报告导出；这是 Phase 3 的客观题垂直切片，不代表 Phase 2 或 Phase 3 已完成。 - 自动化、CI、Compose 故障验收和容量演练的模型执行都只使用 Mock；根据层级使用临时 SQLite 或隔离 PostgreSQL 16/Redis 7，不访问真实模型服务，也不产生模型费用。 - Phase 2 仍为 `in_progress`。P2-01 已完整交付：`P2-local-control-plane-v2` 在 clean commit `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 次 warm-up + 5 次 measured trial，23/23 SLO 与逐轮硬门禁全部通过，容量模型为 `qualified`；该实现的 [GitHub Actions run 33146681285](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档收尾 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run 33150080341](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 @@ -34,7 +35,7 @@ LLMBenchLab 是一个面向个人开发者与研究人员的轻量级 LLM 评测 - **可解释指标**：严格总分 `score`、完成率 `completion_rate` 和已回答准确率 `answered_accuracy` 分开呈现，避免把缺失回答隐藏在成功样本中。 - **可靠任务执行基础**：API 先提交 Run，再 best-effort 发送 Redis Streams 通知；独立 Worker 以数据库时间、租约和 fencing token 领取任务，并通过数据库扫描从通知丢失或进程故障中恢复；大快照加载移出事件循环，已领取 Run 在物化题目时仍可续租。 - **幂等与恢复**：同一 Run/Question 只有一条计分证据；租约心跳、有限 attempt、退避、取消、过期接管和 dead-letter 都由数据库裁决，Redis 不是状态数据库。 -- **数据库权威治理**：Web/API admission 把版本化完整 policy 冻结进 Run；global/provider/model/run 四层并发、RPM/TPM 和累计预算在固定锁序中共同裁决，backlog 满时在提交前稳定拒绝，Token/cost hard limit 缺少显式上界或价格时 fail closed。 +- **数据库权威治理**：Web/API admission 把版本化完整 policy 冻结进 Run；global/provider/model/run 四层并发、RPM/TPM 和累计预算在固定锁序中共同裁决，backlog 满时在提交前稳定拒绝，Token/cost hard limit 缺少显式上界或价格时 fail closed。非显式输入估算不会冒充 hard reservation；actual usage 仍保留，只有实际用量超过显式预留才触发对应 overdraw。 - **逐 Provider attempt 账本与公平调度**：每次 HTTP attempt 先 reserve、再持久化 `send_started`、最后 actual/conservative settlement；可证明未发送的 release 保留终态 ledger，另起 generation 并重试当前未发送 ordinal，不重置之前已发送的 HTTP retry。Worker 每个 lease 只新增有界 question quantum，按最久未获服务顺序 cooperative yield，不把让出误计为失败。 - **可审计观测与受控保留**：typed、应用 append-only audit 以稳定 event key 去重；`/tasks/history` 在同一读取快照中校验 retained audit 后给出 counters 与 Run latency，`/metrics/prometheus` 用固定 gauge/enum label、硬样本上限和进程内 single-flight 暴露同源快照。Worker generation 只在真实 scan/claim/lease-heartbeat/progress 后按数据库 UTC 合并刷新，dependency probe 仍明确不检查主循环。八条 Prometheus 规则附固定 Runbook；`llmbenchlab-audit-retention` 提供 canonical JSONL archive、离线 verify、reconcile、精确 restore/delete，默认不删除且不把普通 hash 冒充 WORM。Run created/finished、credential audit 和逐题 Provider 元数据继续遵守非秘密边界。 - **可复现记录**：持久化模型参数、Prompt、Benchmark Hash、协议版本、代码 commit（可用时）、raw response、parsed answer、参考答案快照和逐题评分。 @@ -77,7 +78,7 @@ flowchart LR WorkerEnv[Worker 环境变量 / 旧配置] -.->|兼容读取| Adapters ``` -API 创建 managed Run 时先在数据库锁内检查 backlog、冻结 active policy 与 Run override，再提交数据库、best-effort 发送 Redis 通知并返回 `202`；通知失败不回滚数据库事实。Worker 优先从数据库对账，并可消费重复 Redis 消息；每次写入都校验当前租约 owner/token，每个 Provider HTTP attempt 由数据库 ledger 单独 admission/结算，恢复时跳过已有 Response。数据库因此是唯一事实来源，Redis、日志、指标和 Worker 内存都不能覆盖 Run 状态。前端轮询 Run，进入 `completed`、`failed` 或 `cancelled` 终态后停止。详细设计见 [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)，治理语义见 [`ADR-0009`](docs/decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、交付边界修正 [`ADR-0010`](docs/decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) 和 pre-send retry generation 修正 [`ADR-0011`](docs/decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)，评分语义见 [`docs/BENCHMARK_PROTOCOL.md`](docs/BENCHMARK_PROTOCOL.md)。 +API 创建 managed Run 时先在数据库锁内检查 backlog、冻结 active policy 与 Run override，再提交数据库、best-effort 发送 Redis 通知并返回 `202`；通知失败不回滚数据库事实。Worker 优先从数据库对账，并可消费重复 Redis 消息；每次写入都校验当前租约 owner/token，每个 Provider HTTP attempt 由数据库 ledger 单独 admission/结算，恢复时跳过已有 Response。数据库因此是唯一事实来源，Redis、日志、指标和 Worker 内存都不能覆盖 Run 状态。前端轮询 Run，进入 `completed`、`failed` 或 `cancelled` 终态后停止。详细设计见 [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)，治理语义见 [`ADR-0009`](docs/decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、交付边界修正 [`ADR-0010`](docs/decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、pre-send retry generation 修正 [`ADR-0011`](docs/decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md) 和 observational reservation 修正 [`ADR-0018`](docs/decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md)，评分语义见 [`docs/BENCHMARK_PROTOCOL.md`](docs/BENCHMARK_PROTOCOL.md)。 任务投递是 at-least-once，本地 Response、ledger 状态转换和聚合是幂等的；这不等于 Provider exactly-once。若 Worker 在 `send_started` 后崩溃，本地会保守结算并最终释放 admission permit，但远端幽灵请求可能仍在运行；若 Provider 已响应而本地 Response 尚未提交，接管 Worker 还可能再次调用并产生额外费用。本地 consumed 数是保守预算证据，不是 Provider 账单真值。 @@ -373,7 +374,7 @@ uv run python -m app.db.import_sqlite \ | `3` | `committed_but_verification_failed` | 目标已提交完整 precommit 快照，但 postcommit 验证或报告失败；停止服务并独立对账 | | `4` | `commit_outcome_unknown` | PostgreSQL 未确认 COMMIT；原子事务意味着目标可能为空，也可能已完整提交 | -退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、13 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。当前 repair head `20260829_0006` 降到 `0005` 不删除对象；继续跨过 `20260828_0005` 时，只要 `worker_processes` 有事实就会拒绝。先停止 Worker、保存必要事实并显式清空该表后，才能进入 `0005 → 0004`，而 `0004` 原有 ledger/audit downgrade guard 仍继续生效。只有隔离空库用于完整降级/升级往返，处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 +退出码 `3` 或 `4` 后**禁止盲目重试**。应先隔离目标，检查 Alembic head、13 表行数/主键集/canonical hash 和工具已输出的对账证据；非空目标会拒绝再次导入。工具不提供 PostgreSQL → SQLite 反向同步，回滚依赖保留的 SQLite 源/备份或单独验证的导出流程。当前 head `20260830_0007` 降到 `0006` 会按旧谓词重算 `governance_scopes.overdrawn`，不删除 ledger 或 actual usage；`0006 → 0005` 不删除索引对象。继续跨过 `20260828_0005` 时，只要 `worker_processes` 有事实就会拒绝。先停止 Worker、保存必要事实并显式清空该表后，才能进入 `0005 → 0004`，而 `0004` 原有 ledger/audit downgrade guard 仍继续生效。只有隔离空库用于完整降级/升级往返，处理见 [`docs/OPERATIONS.md`](docs/OPERATIONS.md)。 ## Audit retention 维护 diff --git a/backend/alembic/versions/20260830_0007_repair_observational_overdraw.py b/backend/alembic/versions/20260830_0007_repair_observational_overdraw.py new file mode 100644 index 0000000..cdd7b05 --- /dev/null +++ b/backend/alembic/versions/20260830_0007_repair_observational_overdraw.py @@ -0,0 +1,145 @@ +"""Repair overdraw flags created from observational input-token estimates. + +Revision ID: 20260830_0007 +Revises: 20260829_0006 +Create Date: 2026-08-30 00:00:00 UTC +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "20260830_0007" +down_revision: str | None = "20260829_0006" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_ACTIVE_STATES = ("reserved", "send_started") +_SETTLED_STATES = ("settled_actual", "settled_conservative") + + +def _tables() -> tuple[sa.TableClause, sa.TableClause, sa.TableClause]: + scopes = sa.table( + "governance_scopes", + sa.column("id", sa.String()), + sa.column("overdrawn", sa.Boolean()), + ) + reservations = sa.table( + "provider_call_reservations", + sa.column("state", sa.String()), + sa.column("run_id", sa.String()), + sa.column("global_scope_id", sa.String()), + sa.column("provider_scope_id", sa.String()), + sa.column("model_scope_id", sa.String()), + sa.column("run_scope_id", sa.String()), + sa.column("reserved_input_tokens", sa.BigInteger()), + sa.column("reserved_output_tokens", sa.BigInteger()), + sa.column("reserved_cost_usd", sa.Numeric(20, 8)), + sa.column("actual_input_tokens", sa.BigInteger()), + sa.column("actual_output_tokens", sa.BigInteger()), + sa.column("actual_cost_usd", sa.Numeric(20, 8)), + ) + runs = sa.table( + "evaluation_runs", + sa.column("id", sa.String()), + sa.column("input_token_reservation", sa.BigInteger()), + ) + return scopes, reservations, runs + + +def _lock_and_require_no_active_reservations(reservations: sa.TableClause) -> None: + connection = op.get_bind() + if connection.dialect.name == "sqlite": + connection.exec_driver_sql("BEGIN IMMEDIATE") + elif connection.dialect.name == "postgresql": + connection.exec_driver_sql( + "LOCK TABLE governance_scopes, provider_call_reservations, evaluation_runs " + "IN ACCESS EXCLUSIVE MODE" + ) + active_count = connection.scalar( + sa.select(sa.func.count()) + .select_from(reservations) + .where(reservations.c.state.in_(_ACTIVE_STATES)) + ) + if int(active_count or 0) != 0: + raise RuntimeError( + "Cannot repair governance overdraw semantics while Provider reservations are active" + ) + + +def _overdraw_expression( + scopes: sa.TableClause, + reservations: sa.TableClause, + runs: sa.TableClause, + *, + require_explicit_input: bool, +) -> sa.Exists: + scope_matches = sa.or_( + reservations.c.global_scope_id == scopes.c.id, + reservations.c.provider_scope_id == scopes.c.id, + reservations.c.model_scope_id == scopes.c.id, + reservations.c.run_scope_id == scopes.c.id, + ) + explicit_input = sa.or_( + reservations.c.run_id.is_(None), + sa.exists( + sa.select(1) + .select_from(runs) + .where( + runs.c.id == reservations.c.run_id, + runs.c.input_token_reservation.is_not(None), + ) + ), + ) + input_overdraw = sa.and_( + reservations.c.reserved_input_tokens.is_not(None), + reservations.c.actual_input_tokens.is_not(None), + reservations.c.actual_input_tokens > reservations.c.reserved_input_tokens, + ) + cost_overdraw = sa.and_( + reservations.c.reserved_cost_usd.is_not(None), + reservations.c.actual_cost_usd.is_not(None), + reservations.c.actual_cost_usd > reservations.c.reserved_cost_usd, + ) + if require_explicit_input: + input_overdraw = sa.and_(explicit_input, input_overdraw) + cost_overdraw = sa.and_(explicit_input, cost_overdraw) + output_overdraw = sa.and_( + reservations.c.reserved_output_tokens.is_not(None), + reservations.c.actual_output_tokens.is_not(None), + reservations.c.actual_output_tokens > reservations.c.reserved_output_tokens, + ) + return sa.exists( + sa.select(1) + .select_from(reservations) + .where( + reservations.c.state.in_(_SETTLED_STATES), + scope_matches, + sa.or_(input_overdraw, output_overdraw, cost_overdraw), + ) + ) + + +def _recompute_overdrawn(*, require_explicit_input: bool) -> None: + scopes, reservations, runs = _tables() + _lock_and_require_no_active_reservations(reservations) + op.get_bind().execute( + scopes.update().values( + overdrawn=_overdraw_expression( + scopes, + reservations, + runs, + require_explicit_input=require_explicit_input, + ) + ) + ) + + +def upgrade() -> None: + _recompute_overdrawn(require_explicit_input=True) + + +def downgrade() -> None: + _recompute_overdrawn(require_explicit_input=False) diff --git a/backend/app/db/import_sqlite.py b/backend/app/db/import_sqlite.py index bd45d14..90966ad 100644 --- a/backend/app/db/import_sqlite.py +++ b/backend/app/db/import_sqlite.py @@ -311,14 +311,21 @@ def _ledger_timestamp(value: Any) -> datetime: return value.astimezone(UTC) -def _reservation_is_overdrawn(reservation: Mapping[str, Any], state: str) -> bool: +def _reservation_is_overdrawn( + reservation: Mapping[str, Any], + state: str, + *, + input_reservation_is_explicit: bool, +) -> bool: if state not in _SETTLED_RESERVATION_STATES: return False - for reserved_name, actual_name in ( - ("reserved_input_tokens", "actual_input_tokens"), - ("reserved_output_tokens", "actual_output_tokens"), - ("reserved_cost_usd", "actual_cost_usd"), + for reserved_name, actual_name, requires_explicit_input in ( + ("reserved_input_tokens", "actual_input_tokens", True), + ("reserved_output_tokens", "actual_output_tokens", False), + ("reserved_cost_usd", "actual_cost_usd", True), ): + if requires_explicit_input and not input_reservation_is_explicit: + continue reserved = reservation[reserved_name] actual = reservation[actual_name] if ( @@ -335,6 +342,7 @@ def _accumulate_scope_ledger( reservation: Mapping[str, Any], *, state: str, + input_reservation_is_explicit: bool, ) -> None: if state in _ACTIVE_RESERVATION_STATES: counters.active_reservations += 1 @@ -352,6 +360,7 @@ def _accumulate_scope_ledger( counters.overdrawn = counters.overdrawn or _reservation_is_overdrawn( reservation, state, + input_reservation_is_explicit=input_reservation_is_explicit, ) @@ -379,11 +388,16 @@ def _validate_governance_materializations(connection: Connection) -> None: scopes = Base.metadata.tables["governance_scopes"] buckets = Base.metadata.tables["governance_minute_buckets"] reservations = Base.metadata.tables["provider_call_reservations"] + runs = Base.metadata.tables["evaluation_runs"] scope_rows = tuple(connection.execute(sa.select(scopes).order_by(scopes.c.id)).mappings()) bucket_rows = tuple(connection.execute(sa.select(buckets).order_by(buckets.c.id)).mappings()) reservation_rows = tuple( connection.execute(sa.select(reservations).order_by(reservations.c.id)).mappings() ) + run_input_reservations = { + str(row.id): row.input_token_reservation + for row in connection.execute(sa.select(runs.c.id, runs.c.input_token_reservation)) + } scopes_by_id = {str(row["id"]): row for row in scope_rows} expected_scopes = {scope_id: _ScopeLedgerCounters() for scope_id in scopes_by_id} @@ -400,6 +414,11 @@ def _validate_governance_materializations(connection: Connection) -> None: for reservation in reservation_rows: state = _enum_text(reservation["state"]) + run_id = reservation["run_id"] + input_reservation_is_explicit = run_id is None or ( + str(run_id) in run_input_reservations + and run_input_reservations[str(run_id)] is not None + ) for reference_name, expected_scope_type in _SCOPE_REFERENCE_TYPES: raw_scope_id = reservation[reference_name] if raw_scope_id is None and reference_name == "run_scope_id": @@ -416,6 +435,7 @@ def _validate_governance_materializations(connection: Connection) -> None: expected_scopes[scope_id], reservation, state=state, + input_reservation_is_explicit=input_reservation_is_explicit, ) bucket_key = ( scope_id, diff --git a/backend/app/db/prepare_migrations.py b/backend/app/db/prepare_migrations.py index 77093b1..79b14b8 100644 --- a/backend/app/db/prepare_migrations.py +++ b/backend/app/db/prepare_migrations.py @@ -32,6 +32,7 @@ CREDENTIAL_REVISION = "20260827_0003" GOVERNANCE_REVISION = "20260827_0004" WORKER_PROGRESS_REVISION = "20260828_0005" +INDEX_REPAIR_REVISION = "20260829_0006" _WORKER_PROGRESS_TABLES = {"worker_processes"} _WORKER_PROGRESS_INDEXES = { @@ -872,6 +873,7 @@ def prepare_database( CREDENTIAL_REVISION, GOVERNANCE_REVISION, WORKER_PROGRESS_REVISION, + INDEX_REPAIR_REVISION, ) historical_heads = {(revision,) for revision in historical_revisions} @@ -919,7 +921,10 @@ def prepare_database( source_revision: str if current_heads in historical_heads: source_revision = current_heads[0] - if not sqlite_locked and source_revision != WORKER_PROGRESS_REVISION: + if not sqlite_locked and source_revision not in { + WORKER_PROGRESS_REVISION, + INDEX_REPAIR_REVISION, + }: return PreparationResult(action="versioned") expected_differences = { LEGACY_REVISION: _LEGACY_DIFFERENCES, @@ -928,6 +933,7 @@ def prepare_database( CREDENTIAL_REVISION: _CREDENTIAL_DIFFERENCES, GOVERNANCE_REVISION: _WORKER_PROGRESS_DIFFERENCES, WORKER_PROGRESS_REVISION: (), + INDEX_REPAIR_REVISION: (), }[source_revision] differences = _schema_differences(connection) expected_missing_index_names: frozenset[str] = frozenset() @@ -975,6 +981,7 @@ def prepare_database( CREDENTIAL_REVISION: "versioned_credentials", GOVERNANCE_REVISION: "versioned_governance", WORKER_PROGRESS_REVISION: "versioned_worker_progress", + INDEX_REPAIR_REVISION: "versioned_index_repair", }[source_revision] else: if not sqlite_locked: @@ -984,9 +991,9 @@ def prepare_database( ) differences = _schema_differences(connection) if not differences: - target_revision = "head" - action = "stamped_current" - source_revision = head_revision + target_revision = INDEX_REPAIR_REVISION + action = "stamped_index_repair" + source_revision = INDEX_REPAIR_REVISION elif differences == _PHASE_1_DIFFERENCES: target_revision = PHASE_1_REVISION action = "stamped_phase1" @@ -1066,9 +1073,9 @@ def main() -> int: "Adopted a verified Web-credential SQLite schema at " f"{result.stamped_revision}; backup: {result.backup_path}" ) - elif result.action == "stamped_current": + elif result.action == "stamped_index_repair": print( - "Adopted a verified current SQLite schema at " + "Adopted a verified current SQLite schema before the overdraw data repair at " f"{result.stamped_revision}; backup: {result.backup_path}" ) elif result.action == "stamped_governance": @@ -1106,6 +1113,11 @@ def main() -> int: "Verified a versioned Worker-progress SQLite schema before upgrade; " f"backup: {result.backup_path}" ) + elif result.action == "versioned_index_repair": + print( + "Verified a versioned index-repair SQLite schema before upgrade; " + f"backup: {result.backup_path}" + ) return 0 diff --git a/backend/app/governance/audit_archive.py b/backend/app/governance/audit_archive.py index 7d407d5..1543e8e 100644 --- a/backend/app/governance/audit_archive.py +++ b/backend/app/governance/audit_archive.py @@ -34,6 +34,7 @@ { "20260828_0005", "20260829_0006", + "20260830_0007", } ) ARCHIVE_V1_RETENTION_VALUES = ("operational", "security") diff --git a/backend/app/governance/repository.py b/backend/app/governance/repository.py index 4e57943..ec885b5 100644 --- a/backend/app/governance/repository.py +++ b/backend/app/governance/repository.py @@ -601,7 +601,6 @@ def question_context( provider_scope: str, lease_owner: str, lease_token: int, - estimated_input_tokens: int | None, reserved_output_tokens: int | None, reserved_cost_usd: Decimal | None, ) -> ProviderAttemptContext: @@ -636,11 +635,6 @@ def question_context( ) if run is None: raise GovernanceFenceLost("governance_lease_fence_lost") - input_reservation = ( - run.input_token_reservation - if run.input_token_reservation is not None - else estimated_input_tokens - ) return ProviderAttemptContext( run_id=run_id, question_id=question_id, @@ -649,13 +643,15 @@ def question_context( lease_token=lease_token, execution_generation=execution.execution_generation, next_provider_attempt=execution.next_provider_attempt, - reserved_input_tokens=_nonnegative_int("reserved_input_tokens", input_reservation), + reserved_input_tokens=_nonnegative_int( + "reserved_input_tokens", run.input_token_reservation + ), reserved_output_tokens=_nonnegative_int( "reserved_output_tokens", reserved_output_tokens ), reserved_cost_usd=( _nonnegative_money("reserved_cost_usd", reserved_cost_usd) - if reserved_cost_usd is not None + if run.input_token_reservation is not None and reserved_cost_usd is not None else None ), ) @@ -1290,22 +1286,48 @@ def _scope_fact_source(): """Expand every immutable ledger row into its three or four scope facts.""" reservation = ProviderCallReservation + run = EvaluationRun + reservation_facts = ( + select( + reservation.global_scope_id.label("global_scope_id"), + reservation.provider_scope_id.label("provider_scope_id"), + reservation.model_scope_id.label("model_scope_id"), + reservation.run_scope_id.label("run_scope_id"), + reservation.state.label("state"), + reservation.policy_id.label("policy_id"), + reservation.window_start.label("window_start"), + reservation.reserved_input_tokens.label("reserved_input_tokens"), + reservation.reserved_output_tokens.label("reserved_output_tokens"), + reservation.reserved_cost_usd.label("reserved_cost_usd"), + reservation.actual_input_tokens.label("actual_input_tokens"), + reservation.actual_output_tokens.label("actual_output_tokens"), + reservation.actual_cost_usd.label("actual_cost_usd"), + or_( + reservation.run_id.is_(None), + run.input_token_reservation.is_not(None), + ).label("input_reservation_is_explicit"), + ) + .select_from(reservation) + .outerjoin(run, run.id == reservation.run_id) + .subquery() + ) columns = ( - reservation.state.label("state"), - reservation.policy_id.label("policy_id"), - reservation.window_start.label("window_start"), - reservation.reserved_input_tokens.label("reserved_input_tokens"), - reservation.reserved_output_tokens.label("reserved_output_tokens"), - reservation.reserved_cost_usd.label("reserved_cost_usd"), - reservation.actual_input_tokens.label("actual_input_tokens"), - reservation.actual_output_tokens.label("actual_output_tokens"), - reservation.actual_cost_usd.label("actual_cost_usd"), + reservation_facts.c.state, + reservation_facts.c.policy_id, + reservation_facts.c.window_start, + reservation_facts.c.reserved_input_tokens, + reservation_facts.c.reserved_output_tokens, + reservation_facts.c.reserved_cost_usd, + reservation_facts.c.actual_input_tokens, + reservation_facts.c.actual_output_tokens, + reservation_facts.c.actual_cost_usd, + reservation_facts.c.input_reservation_is_explicit, ) scope_columns = ( - reservation.global_scope_id, - reservation.provider_scope_id, - reservation.model_scope_id, - reservation.run_scope_id, + reservation_facts.c.global_scope_id, + reservation_facts.c.provider_scope_id, + reservation_facts.c.model_scope_id, + reservation_facts.c.run_scope_id, ) return union_all( *( @@ -1320,6 +1342,7 @@ def _scope_fact_aggregates(facts: Any) -> tuple[Any, ...]: facts.c.state.in_(_SETTLED_STATES), or_( and_( + facts.c.input_reservation_is_explicit, facts.c.reserved_input_tokens.is_not(None), facts.c.actual_input_tokens.is_not(None), facts.c.actual_input_tokens > facts.c.reserved_input_tokens, @@ -1330,6 +1353,7 @@ def _scope_fact_aggregates(facts: Any) -> tuple[Any, ...]: facts.c.actual_output_tokens > facts.c.reserved_output_tokens, ), and_( + facts.c.input_reservation_is_explicit, facts.c.reserved_cost_usd.is_not(None), facts.c.actual_cost_usd.is_not(None), facts.c.actual_cost_usd > facts.c.reserved_cost_usd, @@ -1777,6 +1801,7 @@ def _settle_locked( conservative = ( force_conservative or reconciled or input_tokens is None or output_tokens is None ) + input_reservation_is_explicit = self._input_reservation_is_explicit(session, reservation) for scope in scopes: scope.active_reservations -= 1 scope.reserved_input_tokens -= reserved_input @@ -1787,7 +1812,8 @@ def _settle_locked( scope.consumed_cost_usd = _money(scope.consumed_cost_usd) + _money(settled_cost) if ( ( - reservation.reserved_input_tokens is not None + input_reservation_is_explicit + and reservation.reserved_input_tokens is not None and settled_input is not None and settled_input > reserved_input ) @@ -1797,7 +1823,8 @@ def _settle_locked( and settled_output > reserved_output ) or ( - reservation.reserved_cost_usd is not None + input_reservation_is_explicit + and reservation.reserved_cost_usd is not None and settled_cost is not None and _money(settled_cost) > reserved_cost ) @@ -1830,6 +1857,24 @@ def _settle_locked( now=now, ) + @staticmethod + def _input_reservation_is_explicit( + session: Session, + reservation: ProviderCallReservation, + ) -> bool: + """Return whether a ledger input reservation came from an explicit hard bound.""" + + if reservation.run_id is None: + return True + row = session.execute( + select(EvaluationRun.id, EvaluationRun.input_token_reservation).where( + EvaluationRun.id == reservation.run_id + ) + ).one_or_none() + if row is None: + raise GovernanceIntegrityError("governance_reservation_run_missing") + return row.input_token_reservation is not None + @staticmethod def _append_settlement_audit( session: Session, diff --git a/backend/app/runners/evaluation_runner.py b/backend/app/runners/evaluation_runner.py index 0a33105..fd2bdbe 100644 --- a/backend/app/runners/evaluation_runner.py +++ b/backend/app/runners/evaluation_runner.py @@ -889,13 +889,8 @@ async def _evaluate_question( # old Runs migrated as explicitly unmanaged. generated = await adapter.generate(messages, config) else: - estimated_input_tokens = self._estimate_input_tokens(messages) reserved_output_tokens = self._reserved_output_tokens(config) - reserved_input_tokens = ( - model.governance.input_token_reservation - if model.governance.input_token_reservation is not None - else estimated_input_tokens - ) + reserved_input_tokens = model.governance.input_token_reservation reserved_cost = self._cost( model, reserved_input_tokens, @@ -910,7 +905,6 @@ async def _evaluate_question( provider_scope=model.governance.provider_scope, lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=estimated_input_tokens, reserved_output_tokens=reserved_output_tokens, reserved_cost_usd=reserved_cost, ) @@ -1064,16 +1058,6 @@ async def _evaluate_question( raise _LeaseUnavailable("response_fence_lost") return disposition - @staticmethod - def _estimate_input_tokens(messages: list[dict[str, str]]) -> int: - """Return an observational UTF-8/protocol estimate, never a hard bound.""" - - encoded_bytes = sum( - len(message["role"].encode("utf-8")) + len(message["content"].encode("utf-8")) + 8 - for message in messages - ) - return max(1, (encoded_bytes + 3) // 4) - @staticmethod def _reserved_output_tokens(generation: Mapping[str, Any]) -> int | None: value = generation.get("max_tokens") diff --git a/backend/tests/integration/test_postgres_leases.py b/backend/tests/integration/test_postgres_leases.py index 8f3059a..f92db81 100644 --- a/backend/tests/integration/test_postgres_leases.py +++ b/backend/tests/integration/test_postgres_leases.py @@ -286,7 +286,6 @@ def _prepare_atomic_governance_attempts( provider_scope=provider_scope_key("mock", None), lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=4, reserved_output_tokens=2, reserved_cost_usd=Decimal("0.10000000"), ) @@ -519,7 +518,6 @@ def test_postgres_governance_concurrency_limits_are_atomic_across_connections( provider_scope=provider_scope_key("mock", None), lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=4, reserved_output_tokens=2, reserved_cost_usd=None, ) diff --git a/backend/tests/test_audit_archive.py b/backend/tests/test_audit_archive.py index 5cc71d7..05fdeb9 100644 --- a/backend/tests/test_audit_archive.py +++ b/backend/tests/test_audit_archive.py @@ -282,15 +282,16 @@ def test_archive_v1_rejects_unsupported_source_revision_on_write_and_read(tmp_pa verify_archive(path) -def test_archive_v1_accepts_schema_equivalent_repair_revision() -> None: +@pytest.mark.parametrize("source_revision", ["20260829_0006", "20260830_0007"]) +def test_archive_v1_accepts_compatible_repair_revision(source_revision: str) -> None: data, _digest = build_archive_bytes( (), cutoff_at=datetime(2026, 1, 1, tzinfo=UTC), - source_alembic_head="20260829_0006", + source_alembic_head=source_revision, has_more_eligible=False, ) - assert b'"source_alembic_head":"20260829_0006"' in data + assert f'"source_alembic_head":"{source_revision}"'.encode() in data def test_frozen_archive_v1_fixture_remains_verifiable(tmp_path) -> None: diff --git a/backend/tests/test_evaluation_runner_reliability.py b/backend/tests/test_evaluation_runner_reliability.py index a7df640..8e30f70 100644 --- a/backend/tests/test_evaluation_runner_reliability.py +++ b/backend/tests/test_evaluation_runner_reliability.py @@ -916,7 +916,7 @@ async def test_managed_question_uses_frozen_context_and_persists_safe_provider_e assert disposition == ResponseDisposition.INSERTED context_call = governance_repository.question_context_calls[0] assert context_call["provider_scope"] == "a" * 64 - assert context_call["estimated_input_tokens"] > 0 + assert "estimated_input_tokens" not in context_call assert context_call["reserved_output_tokens"] == 25 assert context_call["reserved_cost_usd"] == Decimal("0.0003") assert adapter.calls[0][2] == {"attempt_context": context} diff --git a/backend/tests/test_governance.py b/backend/tests/test_governance.py index 333af90..dce698d 100644 --- a/backend/tests/test_governance.py +++ b/backend/tests/test_governance.py @@ -202,7 +202,6 @@ def _context( provider_scope=provider_scope_key("mock", None), lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=4, reserved_output_tokens=2, reserved_cost_usd=Decimal("0.00001000"), ) @@ -516,6 +515,191 @@ def test_attempt_actual_settlement_updates_all_scopes_and_bucket(governance_stor ) +def test_observational_input_usage_does_not_create_hard_overdraw(governance_store) -> None: + clock = FixedDatabaseClock(datetime(2026, 8, 27, 14, 0, tzinfo=UTC)) + repository = GovernanceRepository(governance_store, clock=clock) + _admit(governance_store, repository) + lease = _claim(governance_store, clock) + context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-1", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=None, + reserved_cost_usd=Decimal("0.00010000"), + ) + + assert context.reserved_input_tokens is None + assert context.reserved_cost_usd is None + permit = repository.reserve(context, provider_attempt=1, lease_owner=lease.owner) + repository.mark_send_started(permit, lease_owner=lease.owner) + repository.finish( + permit, + disposition=ProviderAttemptDisposition.SETTLED_ACTUAL, + outcome=ProviderAttemptOutcome.SUCCEEDED, + input_tokens=75, + output_tokens=5, + actual_cost_usd=Decimal("0.00020000"), + ) + + with governance_store() as session: + reservation = session.get(ProviderCallReservation, permit.reservation_id) + assert reservation is not None + assert reservation.reserved_input_tokens is None + assert reservation.reserved_cost_usd is None + assert reservation.actual_input_tokens == 75 + assert reservation.actual_cost_usd == Decimal("0.00020000") + assert all(scope.overdrawn is False for scope in session.scalars(select(GovernanceScope))) + + next_context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-2", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=None, + reserved_cost_usd=None, + ) + next_permit = repository.reserve( + next_context, + provider_attempt=1, + lease_owner=lease.owner, + ) + repository.finish( + next_permit, + disposition=ProviderAttemptDisposition.RELEASED_PRE_SEND, + outcome=ProviderAttemptOutcome.MARK_SEND_FAILED, + input_tokens=None, + output_tokens=None, + actual_cost_usd=None, + ) + + +def test_explicit_input_reservation_overdraw_still_stops_following_attempts( + governance_store, +) -> None: + clock = FixedDatabaseClock(datetime(2026, 8, 27, 14, 0, tzinfo=UTC)) + repository = GovernanceRepository(governance_store, clock=clock) + run = _pending_run("run-explicit-input-overdraw") + run.input_token_reservation = 59 + with governance_store() as session, session.begin(): + repository.admit_run(session, run, provider_type="mock", base_url=None) + lease = _claim( + governance_store, + clock, + run_id=run.id, + owner="worker-explicit-input-overdraw", + ) + context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-1", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=None, + reserved_cost_usd=None, + ) + permit = repository.reserve(context, provider_attempt=1, lease_owner=lease.owner) + repository.mark_send_started(permit, lease_owner=lease.owner) + repository.finish( + permit, + disposition=ProviderAttemptDisposition.SETTLED_ACTUAL, + outcome=ProviderAttemptOutcome.SUCCEEDED, + input_tokens=75, + output_tokens=5, + actual_cost_usd=None, + ) + + with governance_store() as session: + assert all(scope.overdrawn is True for scope in session.scalars(select(GovernanceScope))) + + next_context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-2", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=None, + reserved_cost_usd=None, + ) + with pytest.raises(GovernanceExhausted) as caught: + repository.reserve(next_context, provider_attempt=1, lease_owner=lease.owner) + assert caught.value.code == "governance_global_overdrawn" + + +def test_explicit_output_reservation_overdraw_remains_hard(governance_store) -> None: + clock = FixedDatabaseClock(datetime(2026, 8, 27, 14, 0, tzinfo=UTC)) + repository = GovernanceRepository(governance_store, clock=clock) + _admit(governance_store, repository) + lease = _claim(governance_store, clock) + context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-1", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=4, + reserved_cost_usd=None, + ) + permit = repository.reserve(context, provider_attempt=1, lease_owner=lease.owner) + repository.mark_send_started(permit, lease_owner=lease.owner) + repository.finish( + permit, + disposition=ProviderAttemptDisposition.SETTLED_ACTUAL, + outcome=ProviderAttemptOutcome.SUCCEEDED, + input_tokens=1, + output_tokens=5, + actual_cost_usd=None, + ) + + with governance_store() as session: + assert all(scope.overdrawn is True for scope in session.scalars(select(GovernanceScope))) + + +def test_explicit_cost_reservation_overdraw_remains_hard(governance_store) -> None: + clock = FixedDatabaseClock(datetime(2026, 8, 27, 14, 0, tzinfo=UTC)) + repository = GovernanceRepository(governance_store, clock=clock) + run = _pending_run("run-explicit-cost-overdraw") + run.input_token_reservation = 10 + with governance_store() as session, session.begin(): + repository.admit_run(session, run, provider_type="mock", base_url=None) + lease = _claim( + governance_store, + clock, + run_id=run.id, + owner="worker-explicit-cost-overdraw", + ) + context = repository.question_context( + run_id=lease.run_id, + question_id="question-governance-1", + model_id="model-governance", + provider_scope=provider_scope_key("mock", None), + lease_owner=lease.owner, + lease_token=lease.token, + reserved_output_tokens=10, + reserved_cost_usd=Decimal("0.00010000"), + ) + permit = repository.reserve(context, provider_attempt=1, lease_owner=lease.owner) + repository.mark_send_started(permit, lease_owner=lease.owner) + repository.finish( + permit, + disposition=ProviderAttemptDisposition.SETTLED_ACTUAL, + outcome=ProviderAttemptOutcome.SUCCEEDED, + input_tokens=5, + output_tokens=5, + actual_cost_usd=Decimal("0.00020000"), + ) + + with governance_store() as session: + assert all(scope.overdrawn is True for scope in session.scalars(select(GovernanceScope))) + + def test_pre_send_release_and_expired_send_reconciliation(governance_store) -> None: clock = FixedDatabaseClock(datetime(2026, 8, 27, 14, 0, tzinfo=UTC)) repository = GovernanceRepository(governance_store, clock=clock) @@ -566,9 +750,9 @@ def test_pre_send_release_and_expired_send_reconciliation(governance_store) -> N assert first_row is not None and second_row is not None assert first_row.state == ProviderCallReservationState.RELEASED_PRE_SEND assert second_row.state == ProviderCallReservationState.SETTLED_CONSERVATIVE - assert second_row.actual_input_tokens == 4 + assert second_row.actual_input_tokens is None assert second_row.actual_output_tokens == 2 - assert second_row.actual_cost_usd == Decimal("0.00001000") + assert second_row.actual_cost_usd is None scopes = list(session.scalars(select(GovernanceScope))) assert all(scope.active_reservations == 0 for scope in scopes) assert all(scope.consumed_requests == 1 for scope in scopes) @@ -870,7 +1054,7 @@ def test_send_start_moves_reservation_to_its_database_minute(governance_store) - assert old_bucket.reserved_requests == old_bucket.reserved_input_tokens == 0 assert old_bucket.reserved_output_tokens == 0 assert send_bucket.consumed_requests == 1 - assert send_bucket.reserved_input_tokens == 4 + assert send_bucket.reserved_input_tokens == 0 assert send_bucket.reserved_output_tokens == 2 reservation = session.get(ProviderCallReservation, permit.reservation_id) assert reservation is not None @@ -951,7 +1135,6 @@ def test_cost_only_budget_requires_explicit_finite_token_bounds(governance_store provider_scope=provider_scope_key("mock", None), lease_owner=explicit_lease.owner, lease_token=explicit_lease.token, - estimated_input_tokens=4, reserved_output_tokens=None, reserved_cost_usd=Decimal("0.00001000"), ) @@ -976,7 +1159,6 @@ def test_unknown_unbounded_usage_stays_null_without_false_overdraw(governance_st provider_scope=provider_scope_key("mock", None), lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=4, reserved_output_tokens=None, reserved_cost_usd=None, ) @@ -1009,7 +1191,6 @@ def test_unknown_unbounded_usage_stays_null_without_false_overdraw(governance_st provider_scope=provider_scope_key("mock", None), lease_owner=lease.owner, lease_token=lease.token, - estimated_input_tokens=4, reserved_output_tokens=None, reserved_cost_usd=None, ) diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py index 3477054..b02ad63 100644 --- a/backend/tests/test_migrations.py +++ b/backend/tests/test_migrations.py @@ -16,6 +16,7 @@ from app.db.prepare_migrations import ( CREDENTIAL_REVISION, GOVERNANCE_REVISION, + INDEX_REPAIR_REVISION, LEGACY_REVISION, PHASE_1_REVISION, WORKER_PROGRESS_REVISION, @@ -28,7 +29,7 @@ from app.db.session import create_database_engine BACKEND_ROOT = Path(__file__).resolve().parents[1] -HEAD_REVISION = "20260829_0006" +HEAD_REVISION = "20260830_0007" WEB_CREDENTIAL_REVISION = CREDENTIAL_REVISION RELIABILITY_REVISION = "20260825_0002" @@ -150,10 +151,138 @@ def _insert_active_policy( ) -def _mock_postgresql_0005_preflight( +def _insert_overdraw_repair_fixture( + database_path: Path, + *, + input_token_reservation: int | None, + reserved_input_tokens: int, + reserved_output_tokens: int, + reserved_cost_usd: str, + actual_input_tokens: int, + actual_output_tokens: int, + actual_cost_usd: str, + state: str = "settled_actual", +) -> None: + """Insert one old-semantics ledger projection at revision 0006.""" + + if state not in {"settled_actual", "send_started"}: + raise ValueError("unsupported overdraw repair fixture state") + active = state == "send_started" + settled_actual_input = None if active else actual_input_tokens + settled_actual_output = None if active else actual_output_tokens + settled_actual_cost = None if active else actual_cost_usd + with sqlite3.connect(database_path) as connection: + connection.execute("PRAGMA foreign_keys = ON") + connection.execute( + "INSERT INTO governance_policies (" + "id, version, policy_hash, is_active, backlog_limit, question_quantum, " + "activated_at, created_at) VALUES (" + "'policy-overdraw-repair', 1, ?, 1, 10, 2, " + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + ("a" * 64,), + ) + scope_rows = ( + ("scope-overdraw-global", "global", "global"), + ("scope-overdraw-provider", "provider", "b" * 64), + ("scope-overdraw-model", "model", "model-1"), + ("scope-overdraw-run", "run", "run-1"), + ) + for scope_id, scope_type, scope_key in scope_rows: + connection.execute( + "INSERT INTO governance_scopes (" + "id, scope_type, scope_key, active_reservations, reserved_requests, " + "reserved_input_tokens, reserved_output_tokens, reserved_cost_usd, " + "consumed_requests, consumed_input_tokens, consumed_output_tokens, " + "consumed_cost_usd, overdrawn, created_at, updated_at) VALUES (" + "?, ?, ?, ?, 0, ?, ?, ?, 1, ?, ?, ?, 1, " + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + ( + scope_id, + scope_type, + scope_key, + int(active), + reserved_input_tokens if active else 0, + reserved_output_tokens if active else 0, + reserved_cost_usd if active else "0", + settled_actual_input or 0, + settled_actual_output or 0, + settled_actual_cost or "0", + ), + ) + connection.execute( + "INSERT INTO governance_minute_buckets (" + "id, scope_id, policy_id, window_start, reserved_requests, " + "reserved_input_tokens, reserved_output_tokens, consumed_requests, " + "consumed_input_tokens, consumed_output_tokens, created_at, updated_at) " + "VALUES (?, ?, 'policy-overdraw-repair', '2026-08-30 00:00:00', " + "0, ?, ?, 1, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + ( + f"bucket-{scope_type}-overdraw-repair", + scope_id, + reserved_input_tokens if active else 0, + reserved_output_tokens if active else 0, + settled_actual_input or 0, + settled_actual_output or 0, + ), + ) + connection.execute( + "UPDATE evaluation_runs SET governance_policy_id = 'policy-overdraw-repair', " + "governance_status = 'managed', input_token_reservation = ? WHERE id = 'run-1'", + (input_token_reservation,), + ) + connection.execute( + "INSERT INTO question_executions (" + "id, run_id, question_id, execution_generation, next_provider_attempt, " + "created_at, updated_at) VALUES (" + "'execution-overdraw-repair', 'run-1', 'question-1', 0, 2, " + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" + ) + connection.execute( + "INSERT INTO provider_call_reservations (" + "id, operation_key, policy_id, question_execution_id, run_id, question_id, " + "model_id, global_scope_id, provider_scope_id, model_scope_id, run_scope_id, " + "execution_generation, provider_attempt, lease_owner, lease_token, state, " + "window_start, reserved_input_tokens, reserved_output_tokens, reserved_cost_usd, " + "actual_input_tokens, actual_output_tokens, actual_cost_usd, outcome_code, " + "send_started_at, settled_at, created_at, updated_at) VALUES (" + "'reservation-overdraw-repair', 'run-1:question-1:0:1', " + "'policy-overdraw-repair', 'execution-overdraw-repair', 'run-1', 'question-1', " + "'model-1', 'scope-overdraw-global', 'scope-overdraw-provider', " + "'scope-overdraw-model', 'scope-overdraw-run', 0, 1, 'worker-overdraw-repair', " + "1, ?, '2026-08-30 00:00:00', ?, ?, ?, ?, ?, ?, ?, " + "'2026-08-30 00:00:01', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + ( + state, + reserved_input_tokens, + reserved_output_tokens, + reserved_cost_usd, + settled_actual_input, + settled_actual_output, + settled_actual_cost, + None if active else "succeeded", + None if active else "2026-08-30 00:00:02", + ), + ) + + +def _overdraw_repair_ledger_snapshot(database_path: Path) -> tuple[object, ...]: + with sqlite3.connect(database_path) as connection: + row = connection.execute( + "SELECT id, operation_key, state, reserved_input_tokens, " + "reserved_output_tokens, reserved_cost_usd, actual_input_tokens, " + "actual_output_tokens, actual_cost_usd, outcome_code, send_started_at, " + "settled_at, created_at, updated_at FROM provider_call_reservations " + "WHERE id = 'reservation-overdraw-repair'" + ).fetchone() + assert row is not None + return row + + +def _mock_postgresql_historical_preflight( monkeypatch: pytest.MonkeyPatch, *, differences: tuple[str, ...], + source_revision: str = WORKER_PROGRESS_REVISION, ) -> tuple[Mock, Mock, list[object]]: connection = Mock() connection.dialect.name = "postgresql" @@ -173,7 +302,7 @@ def schema_differences(candidate: object) -> tuple[str, ...]: monkeypatch.setattr( prepare_migrations_module, "database_heads", - lambda _connection: (WORKER_PROGRESS_REVISION,), + lambda _connection: (source_revision,), ) monkeypatch.setattr( prepare_migrations_module, @@ -366,9 +495,11 @@ def test_prepare_adopts_current_unversioned_schema(tmp_path: Path) -> None: result = prepare_database(_database_url(database_path)) - assert result.action == "stamped_current" - assert result.stamped_revision == HEAD_REVISION + assert result.action == "stamped_index_repair" + assert result.stamped_revision == INDEX_REPAIR_REVISION assert result.backup_path is not None and result.backup_path.is_file() + assert _read_heads(database_path) == (INDEX_REPAIR_REVISION,) + _run_alembic(database_path, "upgrade", "head") assert _read_heads(database_path) == (HEAD_REVISION,) @@ -641,6 +772,182 @@ def test_known_governance_index_gap_resumes_after_partial_0006_ddl(tmp_path: Pat _run_alembic(database_path, "check") +def test_observational_overdraw_repair_preserves_ledger_and_is_reversible( + tmp_path: Path, +) -> None: + database_path = tmp_path / "observational-overdraw-repair.db" + _run_alembic(database_path, "upgrade", LEGACY_REVISION) + _insert_legacy_rows(database_path) + _run_alembic(database_path, "upgrade", INDEX_REPAIR_REVISION) + _insert_overdraw_repair_fixture( + database_path, + input_token_reservation=None, + reserved_input_tokens=59, + reserved_output_tokens=128, + reserved_cost_usd="0.00010000", + actual_input_tokens=75, + actual_output_tokens=64, + actual_cost_usd="0.00020000", + ) + ledger_before = _overdraw_repair_ledger_snapshot(database_path) + with sqlite3.connect(database_path) as connection: + counters_before = connection.execute( + "SELECT active_reservations, reserved_requests, reserved_input_tokens, " + "reserved_output_tokens, reserved_cost_usd, consumed_requests, " + "consumed_input_tokens, consumed_output_tokens, consumed_cost_usd " + "FROM governance_scopes ORDER BY id" + ).fetchall() + assert ( + connection.execute("SELECT COUNT(*) FROM governance_scopes WHERE overdrawn").fetchone()[ + 0 + ] + == 4 + ) + + _run_alembic(database_path, "upgrade", "head") + + assert _read_heads(database_path) == (HEAD_REVISION,) + assert _overdraw_repair_ledger_snapshot(database_path) == ledger_before + with sqlite3.connect(database_path) as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM governance_scopes WHERE overdrawn").fetchone()[ + 0 + ] + == 0 + ) + assert ( + connection.execute( + "SELECT active_reservations, reserved_requests, reserved_input_tokens, " + "reserved_output_tokens, reserved_cost_usd, consumed_requests, " + "consumed_input_tokens, consumed_output_tokens, consumed_cost_usd " + "FROM governance_scopes ORDER BY id" + ).fetchall() + == counters_before + ) + + _run_alembic(database_path, "downgrade", INDEX_REPAIR_REVISION) + + assert _read_heads(database_path) == (INDEX_REPAIR_REVISION,) + assert _overdraw_repair_ledger_snapshot(database_path) == ledger_before + with sqlite3.connect(database_path) as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM governance_scopes WHERE overdrawn").fetchone()[ + 0 + ] + == 4 + ) + + +@pytest.mark.parametrize( + ( + "case_name", + "input_token_reservation", + "reserved_input_tokens", + "reserved_output_tokens", + "reserved_cost_usd", + "actual_input_tokens", + "actual_output_tokens", + "actual_cost_usd", + ), + [ + ("input", 59, 59, 128, "0.00020000", 75, 64, "0.00010000"), + ("output", None, 59, 4, "0.00020000", 40, 5, "0.00010000"), + ("cost", 128, 128, 128, "0.00010000", 40, 64, "0.00020000"), + ], +) +def test_overdraw_repair_preserves_explicit_hard_bound_overdraw( + tmp_path: Path, + case_name: str, + input_token_reservation: int | None, + reserved_input_tokens: int, + reserved_output_tokens: int, + reserved_cost_usd: str, + actual_input_tokens: int, + actual_output_tokens: int, + actual_cost_usd: str, +) -> None: + database_path = tmp_path / f"explicit-{case_name}-overdraw-repair.db" + _run_alembic(database_path, "upgrade", LEGACY_REVISION) + _insert_legacy_rows(database_path) + _run_alembic(database_path, "upgrade", INDEX_REPAIR_REVISION) + _insert_overdraw_repair_fixture( + database_path, + input_token_reservation=input_token_reservation, + reserved_input_tokens=reserved_input_tokens, + reserved_output_tokens=reserved_output_tokens, + reserved_cost_usd=reserved_cost_usd, + actual_input_tokens=actual_input_tokens, + actual_output_tokens=actual_output_tokens, + actual_cost_usd=actual_cost_usd, + ) + ledger_before = _overdraw_repair_ledger_snapshot(database_path) + + _run_alembic(database_path, "upgrade", "head") + + assert _read_heads(database_path) == (HEAD_REVISION,) + assert _overdraw_repair_ledger_snapshot(database_path) == ledger_before + with sqlite3.connect(database_path) as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM governance_scopes WHERE overdrawn").fetchone()[ + 0 + ] + == 4 + ) + + +@pytest.mark.parametrize( + ("start_revision", "direction", "target_revision"), + [ + (INDEX_REPAIR_REVISION, "upgrade", "head"), + (HEAD_REVISION, "downgrade", INDEX_REPAIR_REVISION), + ], +) +def test_overdraw_repair_refuses_active_reservations_before_mutation( + tmp_path: Path, + start_revision: str, + direction: str, + target_revision: str, +) -> None: + database_path = tmp_path / f"active-overdraw-{direction}.db" + _run_alembic(database_path, "upgrade", LEGACY_REVISION) + _insert_legacy_rows(database_path) + _run_alembic(database_path, "upgrade", start_revision) + _insert_overdraw_repair_fixture( + database_path, + input_token_reservation=None, + reserved_input_tokens=59, + reserved_output_tokens=128, + reserved_cost_usd="0.00010000", + actual_input_tokens=75, + actual_output_tokens=64, + actual_cost_usd="0.00020000", + state="send_started", + ) + ledger_before = _overdraw_repair_ledger_snapshot(database_path) + with sqlite3.connect(database_path) as connection: + scope_rows_before = connection.execute( + "SELECT id, active_reservations, reserved_input_tokens, " + "reserved_output_tokens, reserved_cost_usd, overdrawn " + "FROM governance_scopes ORDER BY id" + ).fetchall() + + failed = _invoke_alembic(database_path, direction, target_revision) + + assert failed.returncode != 0 + assert "Provider reservations are active" in failed.stderr + assert _read_heads(database_path) == (start_revision,) + assert _overdraw_repair_ledger_snapshot(database_path) == ledger_before + with sqlite3.connect(database_path) as connection: + assert ( + connection.execute( + "SELECT id, active_reservations, reserved_input_tokens, " + "reserved_output_tokens, reserved_cost_usd, overdrawn " + "FROM governance_scopes ORDER BY id" + ).fetchall() + == scope_rows_before + ) + + @pytest.mark.parametrize( ("differences", "expected_row_guard_calls"), [ @@ -667,7 +974,7 @@ def test_prepare_postgresql_0005_accepts_only_canonical_or_known_index_gap( differences: tuple[str, ...], expected_row_guard_calls: int, ) -> None: - engine, connection, metadata_calls = _mock_postgresql_0005_preflight( + engine, connection, metadata_calls = _mock_postgresql_historical_preflight( monkeypatch, differences=differences, ) @@ -684,7 +991,7 @@ def test_prepare_postgresql_0005_accepts_only_canonical_or_known_index_gap( def test_prepare_postgresql_0005_rejects_unknown_drift_with_known_index_gap( monkeypatch: pytest.MonkeyPatch, ) -> None: - engine, connection, metadata_calls = _mock_postgresql_0005_preflight( + engine, connection, metadata_calls = _mock_postgresql_historical_preflight( monkeypatch, differences=( "add_index:evaluation_runs.ix_evaluation_runs_started_at_id", @@ -701,6 +1008,42 @@ def test_prepare_postgresql_0005_rejects_unknown_drift_with_known_index_gap( engine.dispose.assert_called_once_with() +def test_prepare_postgresql_0006_checks_canonical_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, connection, metadata_calls = _mock_postgresql_historical_preflight( + monkeypatch, + differences=(), + source_revision=INDEX_REPAIR_REVISION, + ) + + result = prepare_database("postgresql+psycopg://localhost/llmbenchlab_test") + + assert result.action == "versioned" + assert metadata_calls == [connection] + connection.execute.assert_not_called() + connection.close.assert_called_once_with() + engine.dispose.assert_called_once_with() + + +def test_prepare_postgresql_0006_rejects_metadata_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, connection, metadata_calls = _mock_postgresql_historical_preflight( + monkeypatch, + differences=("add_index:models.ix_models_enabled",), + source_revision=INDEX_REPAIR_REVISION, + ) + + with pytest.raises(SchemaPreparationError, match="historical database"): + prepare_database("postgresql+psycopg://localhost/llmbenchlab_test") + + assert metadata_calls == [connection] + connection.execute.assert_not_called() + connection.close.assert_called_once_with() + engine.dispose.assert_called_once_with() + + def test_known_governance_index_gap_rejects_duplicate_active_policies( tmp_path: Path, ) -> None: diff --git a/backend/tests/test_phase2_acceptance_script.py b/backend/tests/test_phase2_acceptance_script.py index 90b249a..ad33e9e 100644 --- a/backend/tests/test_phase2_acceptance_script.py +++ b/backend/tests/test_phase2_acceptance_script.py @@ -34,7 +34,7 @@ def test_migration_acceptance_retains_both_populated_guards_and_empty_round_trip assert script.PRE_GOVERNANCE_REVISION == "20260827_0003" assert script.GOVERNANCE_REVISION == "20260827_0004" assert script.WORKER_PROGRESS_REVISION == "20260828_0005" - assert script.DATABASE_HEAD_REVISION == "20260829_0006" + assert script.DATABASE_HEAD_REVISION == "20260830_0007" assert "postgres_populated_0005_and_0004_downgrade_guards_with_empty_round_trips" in source assert "Cannot downgrade Worker progress schema" in source assert "Cannot downgrade governance schema" in source diff --git a/backend/tests/test_phase2_capacity_script.py b/backend/tests/test_phase2_capacity_script.py index 6fa2dcf..c88d003 100644 --- a/backend/tests/test_phase2_capacity_script.py +++ b/backend/tests/test_phase2_capacity_script.py @@ -1253,6 +1253,8 @@ def measurement(name: str, workers: int) -> dict[str, object]: def test_governance_reconciliation_sql_independently_rebuilds_both_projections() -> None: sql = script.GOVERNANCE_RECONCILIATION_SQL + assert "LEFT JOIN evaluation_runs AS run ON run.id = reservation.run_id" in sql + assert "input_reservation_is_explicit" in sql for scope_reference in ( "reservation.global_scope_id", "reservation.provider_scope_id", @@ -1974,7 +1976,7 @@ def test_acceptance_requires_both_populated_guards_and_empty_round_trips() -> No assert 'PRE_GOVERNANCE_REVISION = "20260827_0003"' in source assert 'GOVERNANCE_REVISION = "20260827_0004"' in source assert 'WORKER_PROGRESS_REVISION = "20260828_0005"' in source - assert 'DATABASE_HEAD_REVISION = "20260829_0006"' in source + assert 'DATABASE_HEAD_REVISION = "20260830_0007"' in source assert "application_worker_progress_0005_to_0004_guard" in source assert "isolated_governance_0004_to_0003_guard" in source assert "worker_progress_0005_to_0004_round_trip" in source diff --git a/backend/tests/test_sqlite_postgres_import.py b/backend/tests/test_sqlite_postgres_import.py index 9719e50..262e742 100644 --- a/backend/tests/test_sqlite_postgres_import.py +++ b/backend/tests/test_sqlite_postgres_import.py @@ -992,6 +992,38 @@ def test_source_preflight_rejects_reserved_governance_aggregates( engine.dispose() +@pytest.mark.parametrize( + ("input_reservation_is_explicit", "actual_output_tokens", "expected"), + [ + (False, 2, False), + (True, 2, True), + (False, 3, True), + ], +) +def test_importer_overdraw_reconciliation_distinguishes_observations_from_hard_bounds( + input_reservation_is_explicit: bool, + actual_output_tokens: int, + expected: bool, +) -> None: + reservation = { + "reserved_input_tokens": 59, + "reserved_output_tokens": 2, + "reserved_cost_usd": Decimal("0.00010000"), + "actual_input_tokens": 75, + "actual_output_tokens": actual_output_tokens, + "actual_cost_usd": Decimal("0.00020000"), + } + + assert ( + import_sqlite_module._reservation_is_overdrawn( + reservation, + "settled_actual", + input_reservation_is_explicit=input_reservation_is_explicit, + ) + is expected + ) + + @pytest.mark.parametrize( ("table_name", "column_name", "value", "direction", "error_pattern"), [ diff --git a/docs/API.md b/docs/API.md index a4423b6..bcd1c32 100644 --- a/docs/API.md +++ b/docs/API.md @@ -287,7 +287,7 @@ curl -sS http://127.0.0.1:8000/api/v1/tasks/metrics - `runs_with_queue_notification_error`：`last_error=queue_notification_unavailable` 的当前 Run 数。 - `managed_backlog`：当前 `pending/running` 且已冻结治理 policy 的 Run；`governance_delayed` 与 `governance_exhausted` 分别是当前延迟和治理耗尽的 Run 数。 - `active_provider_attempts`：attempt ledger 中仍为 `reserved/send_started` 的行数；它是本地数据库 admission 事实，不承诺 Worker 崩溃后的 Provider 幽灵请求已经停止。 -- `overdrawn_governance_scopes`：已被实际 usage 超出 reservation 的治理 scope 数，不是超额 attempt 数。 +- `overdrawn_governance_scopes`：实际 usage 超过**显式 hard reservation** 的治理 scope 数，不是超额 attempt 数。无显式 `input_token_reservation` 时，输入估算不构成 input/cost overdraw；显式 `max_tokens` 的 output reservation 仍独立生效。 - `total_attempts`、`total_failed_attempts`、`total_dispatches`：Run 表中 lease 取得次数、实际 Run 级失败次数与公平调度 dispatch 次数的总和。 - `worker_expected_processes` 是部署显式声明的最小进程数；`registered/live/stalled/shortfall` 从未停止的 `worker_processes` generation 与同一 DB UTC cutoff 派生，`last_seen_at == cutoff` 仍为 live。 - 五个 `worker_last_*` 字段对未停止 generation 取最近（`MAX`）事实；空集合或尚未发生该事件时为 `null`。响应不包含 generation/worker ID，dependency probe 也不会写入或伪造这些时间。 @@ -803,7 +803,7 @@ curl -sS -X POST http://127.0.0.1:8000/api/v1/benchmarks/reload-demo | `seed` | `42` | 32 位有符号整数或 `null` | | `system_prompt` | `null` | 最长 4000；提供时覆盖 Benchmark system prompt | | `concurrency` | `1` | `1..4`；快照值即实际执行并发度 | -| `input_token_reservation` | `null` | `null` 或严格整数 `1..10000000`；hard TPM/Token/费用启用时必须提供可证明的每题输入预留上界 | +| `input_token_reservation` | `null` | `null` 或严格整数 `1..10000000`；hard TPM/Token/费用启用时必须提供可证明的每题输入预留上界。`null` 时实现不会把 UTF-8/tokenizer 估算写成 hard reservation 或用它触发 input/cost overdraw | | `lifetime_request_budget` | `null` | `null` 或严格整数 `0..1000000000`；覆盖该 Run 的累计请求预算，`0` 拒绝新 attempt | | `lifetime_token_budget` | `null` | `null` 或严格整数 `0..10000000000000`；覆盖该 Run 的累计 Token 预算，`0` 拒绝新 attempt | | `lifetime_cost_budget_usd` | `null` | `null` 或 `0..10000000.00000000` USD，最多 8 位小数；请求接受 JSON number/十进制 string，`RunRead` 的非空响应始终是 JSON string；费用硬边界还要求显式 Token 上界和冻结价格 | @@ -927,7 +927,7 @@ Run 表没有 `credential_id`、ciphertext、nonce 或 keyring 列，Run API 也 | `governance_policy_id` | admission 时冻结的 policy；`null` 只用于 0004 前的 `legacy_unmanaged` Run | | `governance_status` | `legacy_unmanaged`、`managed`、`delayed` 或 `exhausted` | | `governance_reason` / `governance_not_before` | 稳定治理原因与 rate/concurrency 延迟后的最早重新调度数据库时间 | -| `input_token_reservation` | 显式冻结的每题输入 Token 上界；估算值不能替代 hard Token/费用边界 | +| `input_token_reservation` | 显式冻结的每题输入 Token 上界；估算值不能替代 hard Token/费用边界。该字段为 `null` 时 actual input 仍保存，但 attempt 的 input reservation/reserved cost 为空 | | `lifetime_*_budget` | 可选 Run 级 request/Token/USD 累计覆盖值；与冻结 policy 中更严格的适用边界共同生效 | | `lease_owner` | 当前 Worker ID；只在 `running` 且租约活动时非空 | | `lease_token` | 单调递增的 fencing generation；租约释放后保留最后值，防止旧 Worker 写入 | @@ -1065,7 +1065,7 @@ curl -sS -X POST http://127.0.0.1:8000/api/v1/runs \ - `500 governance_integrity_error`：冻结 policy 或治理事实不一致，admission fail closed，并尽力追加固定、无秘密的完整性事件。 - `422`：生成参数、治理 override 或读取超时越界、ID 为空、布尔/浮点冒充治理整数，或出现额外字段。 -Run 一旦提交，不会因随后 Redis 故障、并发/RPM/TPM 饱和而删除。瞬时治理压力让它回到 `pending/delayed` 并设置 `governance_not_before`；确定性 lifetime/pricing/上界失败会聚合已有 Response 后进入 `failed/exhausted`，不会伪造一条题级 0 分 Response。hard Token/TPM/费用启用时，`input_token_reservation` 和有限 `max_tokens` 是必须的显式上界；UTF-8 长度估计不能替代它，费用边界还要求 Model 快照存在 input/output USD 价格。 +Run 一旦提交，不会因随后 Redis 故障、并发/RPM/TPM 饱和而删除。瞬时治理压力让它回到 `pending/delayed` 并设置 `governance_not_before`；确定性 lifetime/pricing/上界失败会聚合已有 Response 后进入 `failed/exhausted`，不会伪造一条题级 0 分 Response。hard Token/TPM/费用启用时，`input_token_reservation` 和有限 `max_tokens` 是必须的显式上界；UTF-8 长度估计不能替代它，费用边界还要求 Model 快照存在 input/output USD 价格。Provider actual usage 始终保存；只有超过显式 input/output 预留或由完整显式上界和价格派生的 reserved cost 才会产生 `*_overdrawn`。Run Detail 对该类原因显示“实际用量曾被判定超过预留”；这一中性历史措辞也适用于升级前已经终止的 Run，不再把原因等同于 conservative settlement。 ### 6.4 `GET /runs/{run_id}` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5a77147..590f0c9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -176,7 +176,7 @@ Model 的凭据来源是显式状态，而不是从 nullable 字段猜测：`moc Governance policy 是不可变、内容寻址且只有一个 active version 的数据库事实。初始化前 `GET /api/v1/governance/policy` 是纯读取，返回 `404 governance_policy_not_initialized`，不会借查询偷偷 bootstrap。`PUT /api/v1/governance/policy` 是 full-document apply：所有字段都必填，不具备 PATCH 语义；相同内容幂等返回现有版本，重新提交历史内容会重新激活原 ID/version 并追加 activation audit。`policy_hash` 覆盖全部 20 个规范化限制字段，读取和每次 attempt admission 都重算校验。managed Run 创建会在没有 policy 时原子 bootstrap 确定性默认版本，然后在 global admission 锁内检查有限 backlog，把全量 policy、ID/hash、opaque provider scope、`question_quantum` 与恰好四个 Run override（input reservation 及 request/Token/USD lifetime budget）冻结到 execution snapshot；后续 policy 变更不追溯改写已提交 Run。外发前还会比较冻结 override 与 Run 列，防止任一一侧被篡改后绕过硬边界。 -policy 的 `null` 表示关闭该维限制，`0` 表示立即拒绝新 admission。四层 global/provider/model/run 限额全部满足才允许一次 attempt；hard TPM/Token budget 必须同时有显式 `input_token_reservation` 与有限 `max_tokens`，hard cost budget 还必须有冻结 USD 单价，否则在任何 Provider 外发前分别以稳定治理错误 fail closed。治理 USD 公开上限为 `10000000.00000000`，API 用 JSON string 返回 Decimal；PostgreSQL 使用精确 `NUMERIC(20,8)`，该更低的公开上限使 SQLite IEEE-754 相邻间距低于半个 `1e-8` 存储量化单位，保留兼容路径的 8 位往返。完整字段与状态码见 [`API.md`](./API.md)，操作与恢复见 [`OPERATIONS.md`](./OPERATIONS.md)。 +policy 的 `null` 表示关闭该维限制，`0` 表示立即拒绝新 admission。四层 global/provider/model/run 限额全部满足才允许一次 attempt；hard TPM/Token budget 必须同时有显式 `input_token_reservation` 与有限 `max_tokens`，hard cost budget 还必须有冻结 USD 单价，否则在任何 Provider 外发前分别以稳定治理错误 fail closed。未显式提供 input reservation 时，Runner 不把 UTF-8/tokenizer 估算写入 `reserved_input_tokens`，也不据此生成 reserved cost；Provider actual input 仍按原值保存。input/cost overdraw 只由显式 Run input reservation 派生，显式 `max_tokens` 形成的 output reservation 则独立生效。治理 USD 公开上限为 `10000000.00000000`，API 用 JSON string 返回 Decimal；PostgreSQL 使用精确 `NUMERIC(20,8)`，该更低的公开上限使 SQLite IEEE-754 相邻间距低于半个 `1e-8` 存储量化单位，保留兼容路径的 8 位往返。完整字段与状态码见 [`API.md`](./API.md)，操作与恢复见 [`OPERATIONS.md`](./OPERATIONS.md)，修正依据见 [`ADR-0018`](./decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md)。 ### 创建并执行 Run @@ -545,7 +545,7 @@ React 主导航包含独立的 Runs 列表页。该页通过 `GET /runs` 以 20 - `(benchmark_id, external_id)` 唯一；一个 Run 对同一 Question 最多一条 EvaluationResponse。 - 导入 Benchmark 使用整体事务；逐题结果和进度使用短事务，避免把网络请求包在数据库事务中。 - 只有 owner/token 匹配、租约未过期的 Worker 能写 Response、进度、费用、retry 或终态；唯一约束是最后的竞态防线，不替代 fencing。 -- attempt ledger never-delete 且状态单向；四层 scope/minute materialization 必须能由 ledger 重算，检测到高或低漂移时 fail closed 并由对外边界尽力写固定、无损坏值的完整性 audit，不静默修补后继续外发。 +- attempt ledger never-delete 且状态单向；四层 scope/minute materialization 必须能由 ledger 重算，检测到高或低漂移时 fail closed 并由对外边界尽力写固定、无损坏值的完整性 audit，不静默修补后继续外发。历史 managed ledger 的 input/cost overdraw 重算通过 `evaluation_runs.input_token_reservation` 判断预留是否显式；没有关联 Run 的内部 synthetic reservation 仍把调用者提供的值视为显式。 - audit event 的 `event_key` 唯一，同 key 重放必须匹配 event type 与 payload hash；operational/security 默认分别至少保留 90/365 天。append-only 是应用约束，不是密码学 WORM 或数据库管理员防篡改。 - 聚合从已持久化的 Responses 计算，并在 completed、cancelled 或 dead-letter 终态更新前写回 Run。报告仍防御性地从计划题与 Responses 重算唯一主指标，并只用 `metrics_provenance` 标注 Run 字段漂移，保证 summary/groups/responses 不互相矛盾。 @@ -560,7 +560,7 @@ React 主导航包含独立的 Runs 列表页。该页通过 `GET /runs` 以 20 带凭据的目标 DSN 必须通过 `--target-env` 从受控环境读取；`--target` 拒绝 URL password 和 password query。COMMIT 未获得 PostgreSQL 确认时退出 `4`/`commit_outcome_unknown`；由于事务原子性，目标可能为空，也可能是完整的 precommit 快照。COMMIT 已确认但连接收尾、postcommit 快照/对账或报告失败时退出 `3`/`committed_but_verification_failed`；这时目标已提交完整 precommit 快照，不会自动回滚。两种结果都禁止盲目重试，必须保持目标离线，按已输出摘要独立检查目标是空还是完整提交。非空目标会拒绝再次导入，工具也不提供 PostgreSQL 到 SQLite 的反向同步。 -Alembic `20260827_0004` 引入治理/审计表与 Run/Response 字段；`20260828_0005` 增加 `worker_processes` 与 audit retention/exporter 扫描索引；schema-equivalent `20260829_0006` 只条件补齐早期 `0004` 变体缺少的三个 canonical 索引。兼容 preflight 仍以精确 fingerprint、integrity/FK、索引定义和 single-active 数据约束 fail closed。`0006 → 0005` 不删除 canonical 对象；后续两个 downgrade guard 都在第一条有损 DDL 前拒绝可能丢失的事实：0005 拒绝任意 Worker generation，0004 拒绝 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 证据。隔离空数据库分别验证 `0005 ↔ 0004` 与 `0004 ↔ 0003`；已使用环境应优先向前修复，或恢复经核验的旧备份并单独保留新 schema 证据。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 +Alembic `20260827_0004` 引入治理/审计表与 Run/Response 字段；`20260828_0005` 增加 `worker_processes` 与 audit retention/exporter 扫描索引；schema-equivalent `20260829_0006` 只条件补齐早期 `0004` 变体缺少的三个 canonical 索引；当前 data-only head `20260830_0007` 不改 schema、ledger 或 actual usage，只按显式 hard reservation 语义重算 `governance_scopes.overdrawn`。`0007` upgrade/downgrade 均在任何更新前拒绝 `reserved/send_started` active reservation；downgrade 只恢复旧派生谓词。兼容 preflight 仍以精确 fingerprint、integrity/FK、索引定义和 single-active 数据约束 fail closed。`0006 → 0005` 不删除 canonical 对象；后续两个 downgrade guard 都在第一条有损 DDL 前拒绝可能丢失的事实：0005 拒绝任意 Worker generation，0004 拒绝 policy/scope/bucket/question-execution/ledger/audit 或新 Run/Response 证据。隔离空数据库分别验证 `0005 ↔ 0004` 与 `0004 ↔ 0003`；已使用环境应优先向前修复，或恢复经核验的旧备份并单独保留新 schema 证据。完整流程见 [`OPERATIONS.md`](./OPERATIONS.md)。 ## 错误处理与可观察性 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 4aa3035..dad6547 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -338,6 +338,7 @@ Redis 开启 AOF (`appendfsync everysec`) 只改善通知持久性，不是备 - `20260827_0004`：版本化治理 policy、四层 scope/minute counter、question execution、Provider attempt ledger、typed audit、Run fairness/backpressure 字段和 Response Provider metadata。 - `20260828_0005`：Worker progress facts 与 audit retention/exporter 有界扫描索引。 - `20260829_0006`：不改变逻辑 schema 的兼容修复；仅为早期 `0004` 历史变体补齐三个 canonical governance 索引。 +- `20260830_0007`：不改变 schema、ledger 或 Provider actual usage 的数据修复；仅按显式 input/output hard reservation 与由完整上界和价格派生的 reserved cost 语义重算 `governance_scopes.overdrawn`。 本地 SQLite 更新： @@ -345,7 +346,7 @@ Redis 开启 AOF (`appendfsync everysec`) 只改善通知持久性，不是备 make migrate ``` -命令先执行 `app.db.prepare_migrations`，再 `alembic upgrade head`。受支持的未版本化 SQLite 会在严格结构/integrity/FK 检查和一致性备份后 stamp；已知早期 `0004/0005` 变体只有在 revision fingerprint 为 canonical，或仅缺这三个已知索引的非空子集且最多一条 active policy 时，才会备份并交给 `0006` 修复，因此修复 DDL 中断后可安全重入。新近成为 historical 的 PostgreSQL `0005` 同样先做 metadata drift 校验。未知 drift 在写 revision 或 repair DDL 前拒绝。普通 API/Worker 启动只检查 head，不运行 `create_all`、preflight 或 upgrade。 +命令先执行 `app.db.prepare_migrations`，再 `alembic upgrade head`。受支持的未版本化 SQLite 会在严格结构/integrity/FK 检查和一致性备份后 stamp；与 current metadata 一致的未版本化库只 stamp 到 `0006`，确保 data-only `0007` 仍实际执行而不会被跳过。已知早期 `0004/0005` 变体只有在 revision fingerprint 为 canonical，或仅缺这三个已知索引的非空子集且最多一条 active policy 时，才会备份并交给 `0006` 修复，因此修复 DDL 中断后可安全重入。新近成为 historical 的 PostgreSQL `0005/0006` 按各自规则先做 metadata drift 校验；未知 drift 在写 revision 或 repair DDL 前拒绝。`0007` 在更新 materialized flag 前拒绝任何 `reserved/send_started` reservation。普通 API/Worker 启动只检查 head，不运行 `create_all`、preflight 或 upgrade。 Compose 中只有一次性 `migrate` 服务执行： @@ -355,7 +356,7 @@ python -m app.db.prepare_migrations && alembic upgrade head && alembic check `api` 与 `worker` 必须等待 migrate exit 0，然后仅执行 head check。不要把 Alembic 命令加回 API/Worker entrypoint，也不要同时运行多个 migration owner。 -`0006 -> 0005` 是 no-op downgrade，因为三个索引本来就属于 canonical `0004`；`0005 -> 0004` 在 `worker_processes` 有任意 generation fact 时于第一条 DDL 前拒绝；`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时同样拒绝。正常使用后的数据库不能把它们当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0006/0005/0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 +`0007 -> 0006` 只按旧语义重算 `governance_scopes.overdrawn`，保留 reservation、actual usage、Response、audit 与 Run 终态；若有 active reservation 会在更新前拒绝。`0006 -> 0005` 是 no-op downgrade，因为三个索引本来就属于 canonical `0004`；`0005 -> 0004` 在 `worker_processes` 有任意 generation fact 时于第一条 DDL 前拒绝；`0004 -> 0003` 在任何 policy/scope/bucket/question-execution/ledger/audit 行或新 Run/Response 证据存在时同样拒绝。正常使用后的数据库不能把它们当普通代码回滚。`0003 -> 0002` 只要 `model_credentials` 存在任意行也会拒绝，避免静默丢失 Provider Key。`0002 -> 0001` 在发现 `pending` 或 `running` Run 时拒绝；它会删除可靠性元数据但保留核心实体与协议证据。完整 0007/0006/0005/0004 回滚流程见 [OPERATIONS.md](OPERATIONS.md)；schema downgrade 不是 PostgreSQL→SQLite 反向同步，也不恢复 Phase 1 进程内 Runner。 ### 5.3 备份与恢复证据边界 diff --git a/docs/NEXT_TASK.md b/docs/NEXT_TASK.md index 99da59c..b168908 100644 --- a/docs/NEXT_TASK.md +++ b/docs/NEXT_TASK.md @@ -1,10 +1,10 @@ -# 下一任务：实施 P2-07 最小恢复验证切片 +# 下一任务（当前修复完成后）：实施 P2-07 最小恢复验证切片 > 状态：`planned`；P2-07 独立计划、工作日志与 ADR-0016 已建立，功能实现尚未开始 > 对应阶段：[Phase 2 — Reliability](phases/PHASE-2-RELIABILITY.md) > 当前计划：[Phase 2 恢复与运维闭环](plans/2026-08-28-phase-2-recovery-operations.md) > 当前日志：[2026-08-28 P2-07 工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md) -> 当前决策：[ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)，exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) +> 当前决策：[ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)，exact-head amendments [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / [ADR-0018](decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) > 决策基础：[ADR-0005](decisions/ADR-0005-durable-task-execution.md)、[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) ## 当前事实 @@ -13,7 +13,7 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 当前 P2-06 已按 [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 实现： -- P2-06 的 `20260828_0005` 新增 `worker_processes` 和 bounded audit scan indexes；当前维护 head `20260829_0006` 仅修复早期 `0004` 三索引缺口，不改变 13 表/importer/archive 语义。SQLite→PostgreSQL importer 继续做 13 表精确 digest，拒绝 live generation 并复制 stopped/stale facts；表非空时 `0005 -> 0004` 在 DDL 前拒绝。 +- P2-06 的 `20260828_0005` 新增 `worker_processes` 和 bounded audit scan indexes；`20260829_0006` 仅修复早期 `0004` 三索引缺口。当前应用 head `20260830_0007` 只按显式 hard reservation 语义重算 scope overdrawn，不改变 13 表 schema/importer/archive 合同、never-delete ledger 或 actual usage。SQLite→PostgreSQL importer 继续做 13 表精确 digest，拒绝 live generation 并复制 stopped/stale facts；表非空时 `0005 -> 0004` 在 DDL 前拒绝。 - 长运行 Worker 注册唯一 generation，只在真实 scan/claim/lease-heartbeat/progress 后按 DB UTC 合并刷新；`/tasks/metrics` 和 exporter 只公开 expected/registered/live/stalled/shortfall 与聚合时间。dependency probe 固定声明不检查 main-loop progress。 - `GET /api/v1/metrics/prometheus` 输出固定 Prometheus text `0.0.4` gauge，使用一个 DB-time 快照、有界 15 分钟 audit 与 1 小时 latency 窗口、固定 enum label、整次 fail-closed 和每 API 进程 single-flight。 - `deploy/observability/` 提供固定八条告警规则、抓取示例与对应 Operations Runbook；仓库不部署 Prometheus、Alertmanager、OTel 或通知发送器。 @@ -26,7 +26,9 @@ P2-01 已完成，不再重复资格或“重跑碰绿”。P2-06 也已完成 同日又把 `artifacts/benchmarks/` 中已存在的 GPQA-Diamond 与两种 MMLU-Pro profile 通过正式导入 API 加载到默认个人 SQLite，当前为 `4` 个 Benchmark、`24,277` 题，原 Model/Run/Response 保持且没有 Provider 调用。记录 commit `0163b67c00eb59ae59db5f3adb679ad85c799142` 已 push，其精确 SHA run `33266167547` 4/4 成功；该本地数据操作不提交第三方题目、不改变产品实现或路线图，P2-07 的下一任务仍保持不变。 -P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；本次兼容修复已在自动备份后将当前重建库前进到 `0006` 并通过 startup/check。 +当前必须先完成 [observational Token overdraw 修复](plans/2026-08-30-fix-observational-token-overdraw.md)。ADR-0018 已把非显式 input 估算与 hard reservation 分离，并把 data-only head 定为 `20260830_0007`；该 revision 只重算 `governance_scopes.overdrawn`，upgrade/downgrade 均拒绝 active reservation，历史 ledger/actual/Response/Run 终态保持。本地完整验证和当前个人 SQLite 迁移已通过；此维护仍为 `in_progress`，commit/push 与 exact-SHA CI 待完成。在其仓库级闭环前不要启动 P2-07 实施。 + +P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/test/smoke、双方言 migration、真实 PostgreSQL/Redis integration、frontend build、Compose config、Prometheus 规则、clean capacity/acceptance 与技术/安全终审均已通过；原始 evidence 仍不得公开。P2-06 当时未擅自迁移默认用户 SQLite；随后的兼容修复已在自动备份后将当时重建库前进到 `0006` 并通过 startup/check。 ## 已完成：建立 P2-07 工作包 @@ -39,7 +41,7 @@ P2-06 本地与 clean evidence 数值保持记录不变：合并定向、lint/te P2-07 当前为 `planned`、尚未实现。恢复实施时按当前计划从最小只读 verifier 切片开始，之后才依次完成： -- PostgreSQL backup → 空目标 restore → Alembic `20260829_0006` head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 +- PostgreSQL backup → 空目标 restore → Alembic `20260830_0007` exact head → 13 表 count/PK/content fingerprint → managed Run/ledger/audit/Worker stopped-or-stale facts 可读。 - 数据库与数据库外 keyring 独立备份/恢复：匹配 keyring 能解密，缺失/错误 keyring fail closed；日志/证据不得回显 Key 或 envelope。 - Redis 重建/consumer group 恢复、Worker 扩缩、八条告警响应、dead-letter、commit outcome unknown、governance integrity 与 remaining cancel/retry/lease/budget crash matrix 的真实 PostgreSQL/Redis 演练。 - audit archive 作为数据库恢复后的精确校验/补回工具参与演练，但不得把 archive 自身 restore 冒充整库、PITR、RPO/RTO 或 WORM 认证。 @@ -61,5 +63,5 @@ P2-07 当前为 `planned`、尚未实现。恢复实施时按当前计划从最 ## 可直接复制给 Codex 的任务指令 ```text -继续执行 docs/NEXT_TASK.md。P2-07 的 ADR-0016、独立计划和工作日志已建立，不要重复设计或扩大范围。先只实现计划步骤 2 的最小只读 recovery verifier 及其目标测试；完成、复核并记录后再决定是否进入 Redis/Worker 或 rules/harness。自动化只用 Mock/Stub，所有 destructive 操作只针对隔离、精确目标；Phase 2 在 P2-07 完成前保持 in_progress。 +确认 observational Token overdraw 修复已完成本地/远程门禁且当前应用 head 为 20260830_0007 后，再继续执行 docs/NEXT_TASK.md。P2-07 的 ADR-0016、独立计划和工作日志已建立，不要重复设计或扩大范围。先只实现计划步骤 2 的最小只读 recovery verifier 及其目标测试；完成、复核并记录后再决定是否进入 Redis/Worker 或 rules/harness。自动化只用 Mock/Stub，所有 destructive 操作只针对隔离、精确目标；Phase 2 在 P2-07 完成前保持 in_progress。 ``` diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index c1edcb8..ed8cd0f 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -10,7 +10,7 @@ - 容量方法和当前 Mock 基线：[PERFORMANCE.md](PERFORMANCE.md) - 部署、迁移和凭据：[DEPLOYMENT.md](DEPLOYMENT.md) -- 数据库治理语义：[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[交付边界 ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[pre-send retry generation ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md) 和 [可观测性/保留 ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) +- 数据库治理语义：[ADR-0009](decisions/ADR-0009-database-governance-audit-fair-scheduling.md)、[交付边界 ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md)、[pre-send retry generation ADR-0011](decisions/ADR-0011-confirmed-pre-send-release-retry-generation.md)、[可观测性/保留 ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 和 [observational reservation 修正 ADR-0018](decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) - API 字段与错误码：[API.md](API.md) - 安全边界：[SECURITY.md](SECURITY.md) @@ -47,7 +47,7 @@ curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus | `running` / `active_provider_attempts` | 当前 Run lease / ledger 中 active attempt | 超过 policy 或租约失效后不下降属于完整性问题 | | `governance_delayed` | 并发或 UTC fixed-minute RPM/TPM 暂时背压 | 超过预期窗口仍不恢复时检查 policy、数据库时钟和 Worker | | `governance_exhausted` | lifetime request/token/cost、未知上界/价格或 overdrawn 导致终止 | 先核对稳定 reason，再决定新建 Run 或修正未来 policy | -| `overdrawn_governance_scopes` | Provider actual usage 超过 reservation | 非零立即停止扩大负载并核对真实账单/Token 上界 | +| `overdrawn_governance_scopes` | Provider actual usage 超过显式 hard reservation | 非零立即停止扩大负载并核对显式 input/output 上界、由上界和价格计算的 reserved cost 与真实账单；无显式 input bound 的观测估算偏差不应计入 | | `expired_running` | lease 已按数据库时间过期 | 短暂出现可由 reaper 接管；持续非零说明恢复失败 | | `retry_scheduled` / `total_failed_attempts` | Run 级失败等待 / 累计失败预算 | 观察增量，不能用 `attempt_count` 代替失败数 | | `dead_lettered` | 失败预算耗尽的 Run | 任一新增都应检查 Run audit 和已有 Response | @@ -114,9 +114,9 @@ curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus #### Governance overdraw -- **响应**：停止扩大真实 Provider 流量，核对 actual usage、reservation 上界、价格和 Provider 账单；不要把 actual 改回 reservation。 +- **响应**：停止扩大真实 Provider 流量，核对 actual usage、Run 的显式 `input_token_reservation`、显式 `max_tokens`、价格和 Provider 账单；不要把 actual 改回 reservation。若历史事实只有输入估算且 Run 没有显式 input bound，应先核对数据库是否仍停在 `0006`，不要手改 scope flag。 - **证据**：保存 scope 类型、非秘密 consumed/reserved 汇总、policy/Run override hash 和外部账单关联编号。 -- **恢复**：修正未来 Run 的显式 Token/cost bound 或 policy；已 overdrawn 事实保留并由治理规则决定是否允许后续流量。 +- **恢复**：真实 hard overdraw 应修正未来 Run 的显式 Token/cost bound 或 policy，并保留旧事实。旧 `0006` observational input 误判必须在无 active reservation 的维护窗口升级到 `0007`，由 migration 重算 materialized flag；ledger、actual usage、Response 与旧 Run 终态不改写。 - **静默**：不作常规静默；告警消失也不能替代账单对账。 @@ -166,7 +166,7 @@ curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus - 必须有显式有限 `max_tokens`；`null` 的 Provider default 在 hard Token/cost 下会以 `governance_unbounded_output` fail closed。 - cost budget 还要求 USD input/output price；缺失以 `governance_pricing_unknown` fail closed。 - policy 与 Run override 的 USD 有效范围是 `0..10000000.00000000`、最多 8 位小数；API 响应以 JSON string 保留 Decimal。PostgreSQL 使用精确 `NUMERIC(20,8)`；该公开上限使 SQLite 浮点间距低于半个 `1e-8` 存储量化单位，不应绕过 API 直接写入更大金额。 -- actual usage 超出 reservation 时保存 actual、scope 标记 overdrawn 并阻止后续 attempt；已发生的远端消费无法撤销。 +- actual usage 始终按 Provider 返回值保存。只有 actual 超过显式 input/output reservation，或超过由完整显式上界与冻结价格计算的 reserved cost 时，scope 才标记 overdrawn 并阻止后续 attempt；无显式 input reservation 时，输入估算不写入 attempt hard reservation，也不参与 reserved cost 或 input/cost overdraw。显式 `max_tokens` 形成的 output reservation 仍独立生效。 - usage 缺失或远端是否处理不确定时按完整 reservation 保守结算，不按零释放。 `governance_scopes` 和 `provider_call_reservations` 是预算/attempt 事实，`audit_events` 只作历史观察。API 目前没有“剩余预算”端点；如需告警百分比，应由只读运维任务根据 active policy、Run override 及 scope 的 `reserved_* + consumed_*` 计算，并与 Provider 账单独立对账。不要直接更新物化 counter。 @@ -204,7 +204,7 @@ curl -sS http://127.0.0.1:8000/api/v1/metrics/prometheus - `*_request_budget_exhausted` / `*_token_budget_exhausted` / `*_cost_budget_exhausted`：核对冻结 policy/Run override 和 consumed/reserved。 - `governance_input_bound_unknown` / `governance_unbounded_output` / `governance_pricing_unknown`：修正未来 Run 的显式上界或价格；旧 Run 快照不应静默修改。 -- `*_overdrawn`：停止扩大真实流量，核对 Provider actual usage 和账单。 +- `*_overdrawn`：停止扩大真实流量，核对 Provider actual usage、显式 Run input/output reservation、冻结价格、派生 reserved cost 和账单。当前 `0007` 不会把无显式 input bound 的观测估算偏差解释成 overdraw。 - `governance_provider_retry_exhausted`：同一 question execution generation 的 HTTP ordinal 已耗尽，不得通过 yield/restart 重置。 已有 Response 会先聚合进 failed Run，仍可用于诊断，但不能作为 completed 正式比较。 @@ -372,19 +372,25 @@ llmbenchlab-audit-retention restore \ 退出码 `0` 表示已确认成功，`2` 表示在提交前安全失败，`3` 表示提交已成功但后验核验失败，`4` 表示 commit outcome unknown。遇到 `3` 或 `4` 不得盲目重跑 mutation：保留 archive/count/digest，先执行只读 `reconcile` 判定数据库与 archive 的精确关系，再由操作者决定恢复或删除。Archive 与数据库/keyring 备份是不同资产；archive 不含 credential ciphertext/nonce/keyring，也不能单独恢复 stored Provider Key。 -## 10. 0006、0005 与 0004 安全回滚 +## 10. 0007、0006、0005 与 0004 安全回滚 -### 10.0 Governance index compatibility repair 0006 +### 10.1 Observational overdraw 数据修复 0007 + +当前 head `20260830_0007` 不新增表、字段或索引，也不修改 never-delete reservation、Provider actual usage、Response、audit 或 Run 终态。它只关联 managed reservation 对应的 `evaluation_runs.input_token_reservation`，按以下规则重算每个 `governance_scopes.overdrawn`：input/cost 维度只有在 Run 冻结了显式 input reservation 时参与；显式 `max_tokens` 形成的 output reservation 独立参与；没有关联 Run 的内部 synthetic reservation 继续把调用者提供的值视为显式。 + +`0006 -> 0007` 和 `0007 -> 0006` 都会在任何 materialized 更新前拒绝仍为 `reserved` 或 `send_started` 的 attempt。标准维护流程是停止 API/Worker/CLI writer，确认 active reservation 为零并创建一致性备份，再由唯一 migration owner 执行 `make migrate`。直接 `UPDATE governance_scopes`、删除 ledger 或裁剪 actual usage 均不受支持。降级到 `0006` 只按旧谓词重算 flag，让旧应用看到与旧 runtime 一致的 projection；它不会恢复旧代码、删除事实或把历史失败 Run 改成成功。 + +### 10.2 Governance index compatibility repair 0006 revision `20260829_0006` 不增加新的业务字段或表，只为曾执行早期 `0004` 变体的数据库条件补齐 `ix_evaluation_runs_started_at_id`、`ix_evaluation_runs_finished_at_id` 与 `uq_governance_policies_single_active`。`make migrate` 只在 revision fingerprint 为 canonical，或缺失项是这三个索引的非空子集时放行，并先创建 SQLite 一致性备份；这允许在 SQLite repair DDL 部分完成但 marker 仍为 `0005` 时安全重入。PostgreSQL `0005` 也必须通过 metadata drift 校验。若有多条 active policy、错误的同名索引或任何额外 drift，必须人工核对，工具不会自动停用 policy。`0006 -> 0005` 保留这三个 canonical `0004` 索引，因此是有意的 no-op downgrade。 -### 10.1 Worker progress / retention revision 0005 +### 10.3 Worker progress / retention revision 0005 revision `20260828_0005` 增加 `worker_processes` 和 audit retention/exporter 扫描索引。`0005 -> 0004` 在第一条 DDL 前检查 Worker process facts；只要表中存在任何 generation 行就拒绝，以免静默丢失注册、进展或 graceful-stop 证据。安全降级必须先停止 Worker、归档需要保留的 process facts，并由明确的数据生命周期审批清空；正常代码回滚应保留 0005 schema 并优先向前修复。 正式迁移验收同时保留两层独立门禁：populated 0005 数据库拒绝跨过 Worker progress revision且 revision/核心事实不变；隔离空库允许 `0005 -> 0004 -> 0005`。这不会替代下节已有的 populated 0004 governance 拒绝与空库 `0004 -> 0003 -> 0004` 往返。 -### 10.2 Governance/audit revision 0004 +### 10.4 Governance/audit revision 0004 revision `20260827_0004` 增加 policy、scope、minute bucket、question execution、Provider attempt ledger、typed audit、Run governance/fairness 字段及 Response Provider metadata。downgrade guard 在第一条 DDL 前检查数据损失风险。 diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 10247d3..6b8c816 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -14,7 +14,7 @@ `0.1.0` development baseline，REST API 为 `/api/v1`，评测协议为 `llmbenchlab-protocol-v1`；尚未发布正式 Release。 -公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。[ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / `20260829_0006` 数据库兼容修复实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已普通 push，其精确 SHA 的 GitHub Actions [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 四个必需 job 全部成功，因此该维护任务为 `completed`。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 +公开仓库：[`CWNU-Open-Source-Community/LLMBenchLab`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab)，当前开发分支为 `codex/complete-evaluation-workflow`。P2-06 实现 SHA [`9a20676dcf545040782f04c166205d0043345753`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/9a20676dcf545040782f04c166205d0043345753) 已普通 push 并进入 [PR #3](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/3)，其精确 SHA 的 GitHub Actions [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 四个必需 job 全部成功；绑定该 clean SHA 的 capacity 与 9/9 acceptance 也已通过。Evidence closeout 文档 commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA 的 GitHub Actions [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 四个必需 job 全部成功，因此 P2-06 已完成仓库级收尾并标记为 `completed`。[ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / `20260829_0006` 数据库兼容修复实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 已普通 push，其精确 SHA 的 GitHub Actions [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 四个必需 job 全部成功，因此该维护任务为 `completed`。当前工作树正按 [ADR-0018](decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) 修复 observational input estimate 误触发 overdraw，应用与当前个人 SQLite head 均已前进为 data-only `20260830_0007`；本地完整门禁与数据验真已通过。本维护仍为 `in_progress`，因为 commit/push 与 exact-SHA CI 尚未完成，不能宣称仓库级闭环。Phase 2 仍为 `in_progress`；P2-07 已建立 ADR-0016、独立计划和工作日志，状态为 `planned`，功能实现尚未开始。历史 P2-01 位于 [PR #2](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/pull/2)：实现 SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 的 [run `33146681285`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33146681285) 4/4 成功，证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 的 [run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 也已 4/4 成功。 ## 已交付基线 @@ -46,14 +46,14 @@ - [ADR-0015](decisions/ADR-0015-observability-worker-progress-audit-retention.md) 已接受；实现 SHA `9a20676dcf545040782f04c166205d0043345753` 将 Alembic head 扩展到 `20260828_0005`。`worker_processes` 保存 generation 级 DB UTC `started/seen/scan/claim/progress/lease-heartbeat/stop`，主循环只在真实事件后合并刷新；JSON metrics 公开 expected/registered/live/stalled/shortfall 与最近时间，不公开 Worker/generation ID。dependency probe 固定声明 `main_loop_progress=not_checked`。 - `GET /api/v1/metrics/prometheus` 已实现固定 Prometheus text `0.0.4` gauge：一个 DB-time 读快照、15 分钟 typed-audit 窗口、1 小时 Run latency、硬读取上限、固定 enum label、整次 fail-closed 与每 API 进程 single-flight。`deploy/observability/` 提供固定八条规则和安全抓取示例；仓库不部署 Prometheus、Alertmanager 或通知发送器。 - `llmbenchlab-audit-retention archive|verify|reconcile|restore|delete` 已实现 canonical JSONL v1、严格权限/大小/行/schema/hash/rollup 校验、离线 verify、精确 digest 绑定、默认不删除、双方言事务与 commit outcome 分类。Archive 是敏感运维文件，hash 只用于完整性/绑定，不是签名或 WORM，也不替代 P2-07 的数据库+keyring 备份。 -- P2-06 的 `0005` 将 importer 逻辑合同扩展为 13 表精确 count/PK/content digest；当前 source/target 仍必须位于唯一 current head `0006`，13 表语义不变。live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 +- P2-06 的 `0005` 将 importer 逻辑合同扩展为 13 表精确 count/PK/content digest；source/target 必须位于唯一 current head，现为 data-only `0007`，13 表 schema/内容合同不变。live generation 在源 preflight 被拒绝，stopped/stale facts 可复制，终审又补强 committed target canonical integrity postverify。`0005 -> 0004` 在 `worker_processes` 非空时于 DDL 前拒绝，原有 `0004` governance/audit downgrade guard 继续保留。 - 生产日志源已统一治理：应用日志消息必须是无格式参数字面量，结构化字段按白名单和有限数值输出，第三方动态消息固定化且不能通过 allowlisted extra 注入，raw Uvicorn access handler 关闭。Archive 终审补充了 FIFO/非普通文件拒绝及 decode 前行数上限；retention 零行 mutation 仍须 postverify，PostgreSQL mutation 保持 advisory/row lock。 - 上述实现的全部实现门禁已完成：合并定向套件、`make lint`（Ruff 152 files、ESLint、TypeScript）、`make test`（后端 `916 passed, 33 skipped`、前端 `38 passed`）、Mock smoke（`1 passed, 7 deselected`）、临时 PostgreSQL 16/Redis 7 migration/check 与真实 integration（`33 passed, 0 skipped`）、隔离 SQLite migration/check、frontend build、Compose config、八规则 `promtool` 和修复后 76-file staged 技术/安全终审均通过；实现 SHA 已 push，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json` 的 SHA-256 为 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`，9/9 通过，Worker expected/registered/live/stalled/shortfall=`2/2/2/0/0`，cleanup C/V/N 全空。Clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json` 的 SHA-256 为 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`；1W/2W/burst QPS=`7.267474/12.962228/9.333604`、wall=`8.255963/4.628834/6.428385s`，最终 18 Runs/270 Responses/270 question executions/271 reservations/1230 audit，0 question error/drift/duplicate/PEL/lag，Worker expected=2、shortfall=0，cleanup C/V/N/image 全零且 image counters=`1/1/0/0`。两份 evidence 均为 `dirty=false` 并绑定 `9a20676…`；这是 Mock-only、非 SLO。此前 dirty acceptance/capacity 继续作为历史证据保留。Evidence closeout 文档 commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 已 push，精确 SHA run `33165775037` 4/4 成功，P2-06 仓库级收尾完成。P2-06 当时默认用户 SQLite 尚未在 head，直接 `alembic check` 失败后按保护原则未擅自迁移。 ## 2026-08-29 数据库兼容修复（`completed`） - 旧库失败不是 SQLite 损坏，而是 revision=`20260827_0004` 的早期结构变体精确缺少三个后来加入 canonical `0004` 的索引；严格 preflight 因 revision/schema 不一致，在 `0005` 执行前按设计拒绝。 -- [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) 已接受；当前 head 为 schema-equivalent `20260829_0006`。preflight 只接受 canonical `0004/0005`，或仅缺一至三个已知 repair 索引的 fingerprint，以支持 SQLite repair 中断重入；PostgreSQL `0005` 的额外 metadata drift 同样拒绝。完整 schema/integrity/FK、索引定义、single-active policy 数据门禁仍 fail closed。`0006` 首条 DDL 前再次拒绝多条 active policy，并条件补建索引；`0006 -> 0005` 保留 canonical 对象。 +- [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) 已接受；该任务完成时的 head 为 schema-equivalent `20260829_0006`，现作为 `0007` 的历史下游 revision。preflight 只接受 canonical `0004/0005`，或仅缺一至三个已知 repair 索引的 fingerprint，以支持 SQLite repair 中断重入；PostgreSQL `0005` 的额外 metadata drift 同样拒绝。完整 schema/integrity/FK、索引定义、single-active policy 数据门禁仍 fail closed。`0006` 首条 DDL 前再次拒绝多条 active policy，并条件补建索引；`0006 -> 0005` 保留 canonical 对象。 - 真实失败备份副本已无损升级到 `0006` 并通过 integrity/FK/Alembic check；当前重建库由标准 preflight 自动备份后从 canonical `0005` 到 `0006`，startup gate、quick check、FK 和 metadata check 全部通过，业务计数与 Worker facts 保持。 - 本地 migration `52 passed`；最终完整 `make test` 为后端 `927 passed, 33 skipped`、前端 `38 passed`，`make lint`、Mock smoke 和 Compose config 均通过。实现 SHA `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，[run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 的 backend、真实 PostgreSQL/Redis、完整 Compose 与 frontend 四个 job 全部成功；本维护任务完成，不改变 Phase 2/P2-07 状态。 @@ -68,22 +68,31 @@ - `artifacts/benchmarks/` 中三个 Git 忽略 dataset-v1 ZIP 均由当前 Loader 校验通过：GPQA-Diamond `198` 题、MMLU-Pro Direct `12,032` 题、MMLU-Pro Official-CoT `12,032` 题。原始 `dataset-cache` 仍只作为固定来源缓存，没有被当作导入包。 - 在现有 `make dev` 服务无活动 Run 时，先用 SQLite 在线 backup 创建 `0600` 的 `backend/data/llmbenchlab.db.pre-benchmark-import-20260829T173032Z.bak`（SHA-256 `d8c71c44b0ee364030d0053788f34bd985a443ba30a9cd17f1069a9737d10206`），再通过正式 `/api/v1/benchmarks/import` 逐个导入，三个请求均为 `201`。 -- 默认个人 SQLite 现有 `4` 个 Benchmark、`24,277` 道题；逐集持久化题数与 manifest 一致。原有 `1` 个 Model、`1` 个 completed Run 和 `15` 条 Response 不变，active Run 为 `0`；`quick_check=ok`、外键错误 `0`、Alembic head=`20260829_0006`，API 列表/逐集 total 也已对账。 +- 该次导入完成时，默认个人 SQLite 有 `4` 个 Benchmark、`24,277` 道题；逐集持久化题数与 manifest 一致。当时原有 `1` 个 Model、`1` 个 completed Run 和 `15` 条 Response 不变，active Run 为 `0`；`quick_check=ok`、外键错误 `0`、Alembic head=`20260829_0006`，API 列表/逐集 total 也已对账。后续 Run 与 `0007` 维护事实见下节，不回写本条历史快照。 - 目标 Loader/标准转换器离线测试 `40 passed`；未调用真实 Provider，未修改 Schema/API/协议或产品代码。仓库记录 commit [`0163b67c00eb59ae59db5f3adb679ad85c799142`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/0163b67c00eb59ae59db5f3adb679ad85c799142) 已 push，其精确 SHA 的 [run `33266167547`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33266167547) 四个必需 job 全部成功；本地加载维护为 `completed`，不改变 Phase 3/P2-07 状态。 +## 2026-08-30 observational Token overdraw 修复（`in_progress`） + +- 只读核查 Run `2181503c-eab2-4699-bede-db48bd078f95` 发现：15 题完成 7 题后以 `failed/exhausted`、`governance_global_overdrawn` 终止；7 个 Provider attempt 均为 `settled_actual/succeeded`，没有 conservative settlement、429 或 HTTP retry。第七次非 hard 输入估算/reservation 为 59，而 Provider actual 为 75；冻结 policy 与 Run override 的 request/Token/cost hard limit 全为 `null`，因此这是本地语义缺陷，不是 OpenCode Go 套餐额度结算。 +- [ADR-0018](decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) 已接受：只有显式 `evaluation_runs.input_token_reservation` 才构成 input/cost hard reservation；没有显式值时不再写入新的 attempt input reservation/reserved cost，但 Provider actual usage 仍完整保存。显式 input、显式 `max_tokens` output，以及由完整上界和冻结价格派生的 reserved cost 超额仍 fail closed。 +- 应用 Alembic head 已前进到 data-only `20260830_0007`。该 migration 不改 schema、ledger、actual usage、Response、audit 或 Run 终态，只重算 `governance_scopes.overdrawn`；upgrade/downgrade 在任何更新前拒绝 active reservation。前端 overdrawn 文案改为“实际用量曾被判定超过预留”，既适用于新 hard overdraw，也不会误述升级前保留的历史终态。 +- 本地验证已完成：backend `946 passed, 33 skipped`，真实 PostgreSQL+Redis integration `33 passed`，双方言 migration upgrade/downgrade/upgrade/check、`make lint`、frontend `39 passed`/build、Mock smoke `1 passed` 与 Compose config 均通过。当前个人 SQLite 已到 `0007`；四层 scope `overdrawn` 从 4 降为 0，7 Responses/7 ledger、407 input/599 output、13 张业务表行数均保留，`quick_check=ok`、FK=0。未调用真实 Provider。commit/push 和 exact-SHA CI 尚待完成，因此本维护仍为 `in_progress`。 + ## 状态与后续 - P2-06：状态为 `completed`；实现、clean-SHA Compose evidence、实现 commit 与 evidence closeout 文档 commit 的 push 和精确 SHA CI 均已完成。 - 0004 历史索引兼容修复：状态为 `completed`；实现 commit `8fb51b690ae6335b8ef93b3cbe54e039781fb173` 已 push，精确 SHA run `33263405214` 4/4 成功。 - 本地数据恢复与静默启动：状态为 `completed`；实现 commit `5075bdb5e9b53f527a43e5aff7b7d2c7b48c5c9b` 已 push，精确 SHA run `33265171953` 4/4 成功。 - 已下载标准评测集本地加载：状态为 `completed`；三个现有正式 ZIP 已导入并完成本地数据库/API/目标测试验证，`0163b67…` 的 run `33266167547` 4/4 成功。 -- P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、其 exact-head amendment [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。 +- observational Token overdraw 修复：状态为 `in_progress`；目标 head `0007`、本地完整验证和当前库迁移已完成，仅 commit/push 与 exact-SHA CI 远程门禁待完成。 +- P2-07：状态为 `planned`，已建立 [ADR-0016](decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、exact-head amendments [ADR-0017](decisions/ADR-0017-schema-equivalent-governance-index-repair.md) / [ADR-0018](decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md)、[独立计划](plans/2026-08-28-phase-2-recovery-operations.md) 和 [工作日志](worklogs/2026-08-28-phase-2-recovery-operations.md)。PostgreSQL backup/restore、数据库与 keyring 配对恢复、Redis 重建、Worker 扩缩/告警处置和剩余故障矩阵的功能实现尚未开始；P2-06 的 audit archive 自身 restore 不能替代整库恢复认证。P2-07 recovery-manifest-v1 的尚未实施 exact head 现为 `20260830_0007`。 - Phase 3：IFEval、通用 Dataset Plugin SDK、代码题 schema/隔离沙箱、完整分组 UI 和安全红队；Phase 4–6 尚未开始。 ## 已知边界与风险 - SQLite 只用于个人本地单 Worker；多 Worker 证据必须来自 PostgreSQL。Compose 是本地开发/验收拓扑，不是生产 HA。 - Provider 调用不是 exactly-once。Worker 在 Provider response 后本地 commit 前崩溃可能重复上游计算或费用；本地 ledger/Response 幂等只能保证数据库事实不 double-count。 +- 无显式 input reservation 的估算不是费用上界；`0007` 只防止该观测值被误解释为 hard overdraw，不会推断 Provider tokenizer、限制真实账单或替代显式 Token/cost policy。 - fixed UTC minute window 允许边界 burst，不等同平滑 token bucket。Mock capacity 不能推断真实 Provider、生产 SLA 或无限横向扩展。 - trusted-local CLI 按 [ADR-0010](decisions/ADR-0010-phase-2-governance-delivery-boundaries.md) 继续 `legacy_unmanaged`，没有全局 RPM/TPM/USD 硬保证；操作者必须停止常规 API/Worker 并独占数据库。 - audit 是应用 append-only、event-key 幂等且 read 时校验 schema/hash，但数据库管理员仍可修改，不能宣称 WORM。 @@ -103,7 +112,7 @@ | P2-06 Mock smoke | `1 passed, 7 deselected` | 完全离线通过 | | P2-06 真实 PostgreSQL/Redis integration | 临时 PostgreSQL 16/Redis 7 migration/check 后 `33 passed, 0 skipped` | retention advisory/row-lock 与既有 lease/governance/importer 路径通过；首次 cleanup 被安全策略拒绝且未启动容器，修正明确目标后通过；实现 SHA 的远程 integration job 也已成功 | | P2-06 migration | 临时 SQLite/真实 PG 往返与 check 全绿；当时默认用户 SQLite 非 head 的 check 失败后未迁移 | P2-06 当时 head `20260828_0005`；历史证据保持原状 | -| 2026-08-29 DB compatibility repair | migration `52 passed`；最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | current head `20260829_0006`；`8fb51b6…` 的 run `33263405214` 4/4 成功 | +| 2026-08-29 DB compatibility repair | migration `52 passed`；最终完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当时默认库 startup/check 全绿 | 该任务当时 head `20260829_0006`；`8fb51b6…` 的 run `33263405214` 4/4 成功 | | P2-06 build/config | frontend build 成功（保留 662.39 kB chunk warning）；Compose config exit 0 | 从根目录误跑 npm 的失败已记录并用正确目录重跑通过 | | P2-06 Prometheus 规则 | `prom/prometheus:v3.5.0` 中 `promtool check rules` 成功，八条规则全部通过 | 临时容器验证；仓库仍不部署 Prometheus/Alertmanager | | P2-06 dirty acceptance | 9/9；artifact `llmbenchlab-p2-11554c25ec2d/evidence.json`，SHA-256 `d5f058457dbc29875cbac4bc38345b810b5ed556ea538862d309116ceb629fde`，`dirty=true` | Worker `2/2/2/0/0`、`0005`/isolated `0004` populated refusal、两层空库往返、cleanup C/V/N empty | @@ -118,6 +127,7 @@ | P2-01 证据文档收尾 CI | `875f13a…` run `33150080341` 4/4 | 精确文档 SHA 全绿；P2-01 仓库级收尾完成 | | 2026-08-30 本地恢复/静默启动 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、恢复库 digest/quick/FK/head 与真实 API/Web 读取通过 | 默认库恢复 `1/1/15/1/15`；`5075bdb…` 的 run `33265171953` 4/4 成功，不改变 P2-07 | | 2026-08-30 标准评测集本地加载 | 三个 ZIP Loader 校验通过；API 导入 `201/201/201`；数据库/API 对账为 `4` Benchmarks、`24,277` Questions；`quick_check=ok`、FK `0`、head `0006`；目标测试 `40 passed` | 本地加载完成，原 Model/Run/Response 保持；`0163b67…` 的 run `33266167547` 4/4 成功，无 Provider 调用 | +| 2026-08-30 observational overdraw 修复 | backend `946 passed, 33 skipped`；真实 PG+Redis integration `33 passed`；双方言 migration 往返/check、`make lint`、frontend `39 passed`/build、Mock smoke `1 passed`、Compose config 与当前库 backup/migrate/check 通过 | 当前 SQLite head=`20260830_0007`，scope `4→0`，7 Responses/7 ledger/407 input/599 output、13 表行数、quick/FK 保持；无真实 Provider；commit/push/exact-SHA CI 待完成 | | 最新本地 `make lint` | Ruff/format、ESLint、TypeScript 通过 | 本地冻结树通过 | | P2-01 冻结树 `make test` | 后端 `829 passed, 29 skipped`；前端 `38 passed` | v2 实现历史冻结树通过；当前 P2-06 全量见上方独立行 | | P2-01 真实 PostgreSQL/Redis integration | `29/29 passed` | v2 实现历史冻结树通过；当前 P2-06 integration 见上方独立行 | @@ -143,7 +153,8 @@ - [Phase 2 可观测性与审计保留](worklogs/2026-08-28-phase-2-observability-retention.md) - [本地数据恢复与静默启动](worklogs/2026-08-30-restore-data-quiet-startup.md) - [加载已下载的标准评测集](worklogs/2026-08-30-load-downloaded-benchmarks.md) +- [修复 observational Token overdraw](worklogs/2026-08-30-fix-observational-token-overdraw.md) ## 当前任务入口 -[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。本次个人本地 SQLite/启动器维护与标准评测集加载都不改变路线图：P2-06 已完成仓库级收尾；P2-07 工作包已建立、状态为 `planned`，后续从最小只读 recovery verifier 开始实施。Phase 2 与 Phase 3 继续保持 `in_progress`。 +[NEXT_TASK.md](NEXT_TASK.md) 提供后续任务入口。observational Token overdraw 修复的本地验证和个人 SQLite 受控迁移已完成，当前只剩 commit/push 与 exact-SHA CI；仓库级闭环后才恢复 P2-07 最小只读 recovery verifier。P2-06 已完成仓库级收尾，P2-07 仍为 `planned`；Phase 2 与 Phase 3 继续保持 `in_progress`。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ff0555b..8ec2810 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,6 +1,6 @@ # LLMBenchLab Roadmap -> 最后更新：2026-08-28 +> 最后更新：2026-08-30 > 当前阶段：Phase 2 — 可靠性与任务执行；Phase 3 可信本地客观数据切片 > 当前状态：Phase 0–1 `completed`；Phase 2–3 `in_progress`；Phase 4–6 `planned` @@ -30,7 +30,7 @@ | --- | --- | --- | --- | --- | | Phase 0 | 项目治理和架构 | 可执行的需求、架构、协议、ADR 与持续文档流程 | `completed` | [PHASE-0-GOVERNANCE.md](phases/PHASE-0-GOVERNANCE.md) | | Phase 1 | MVP 垂直链路 | Mock 模型到 Run、逐题结果与排行榜的离线闭环 | `completed` | [PHASE-1-MVP.md](phases/PHASE-1-MVP.md) | -| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计、P2-01 与 P2-06 已交付；P2-07 工作包已建立，状态为 `planned`，功能尚未实现 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | +| Phase 2 | 可靠性与任务执行 | 可靠 Worker、治理/审计、P2-01 与 P2-06 已交付；observational overdraw 维护进行中；P2-07 工作包已建立但功能尚未实现 | `in_progress` | [PHASE-2-RELIABILITY.md](phases/PHASE-2-RELIABILITY.md) | | Phase 3 | 标准 Benchmark 与代码评测 | 已有 MMLU-Pro/GPQA 可信本地切片；IFEval、沙箱与完整插件体系待完成 | `in_progress` | [PHASE-3-BENCHMARKS.md](phases/PHASE-3-BENCHMARKS.md) | | Phase 4 | Judge、Arena 与长上下文 | 可校准 Judge、Pairwise Judge、个人 Arena 和长上下文评测 | `planned` | [PHASE-4-JUDGE-ARENA.md](phases/PHASE-4-JUDGE-ARENA.md) | | Phase 5 | Agent、私有与 Live Benchmark | 工具调用轨迹、隔离私有集和持续更新的 Live Benchmark | `planned` | [PHASE-5-AGENT-LIVE.md](phases/PHASE-5-AGENT-LIVE.md) | @@ -181,6 +181,7 @@ - 独立 Worker、原子领取、数据库时间租约、心跳、单调 fencing token、逐题幂等、有限重试、取消、过期接管和 dead-letter；大 Run 快照加载移出事件循环并保持租约心跳，dead-letter 前从持久化 Response 重聚合证据。 - Alembic `0004` 的 policy/scope/minute bucket/question execution/Provider reservation/audit event 六类治理表及 `0005` 的 Worker process/progress 表与 bounded audit indexes；Run/Response 证据字段和 13 表 SQLite→PostgreSQL importer。 - managed Run 冻结 policy/hash 与显式 overrides；global/provider/model/run 四层 concurrency、固定窗口 RPM/TPM、global/run lifetime request/Token/USD budget 和逐 Provider HTTP attempt ledger。 +- 当前 data-only head `20260830_0007` 按 ADR-0018 将观测 input 估算与 hard reservation 分离：无显式 input bound 时不生成 input reservation/reserved cost，actual usage 仍保存；显式 input/output 上界及由完整上界和价格派生的 reserved cost 超额继续 fail closed。本地完整门禁和当前 SQLite 迁移已通过，远程证据仍待完成。 - materialized counter 只作 ledger 投影；counter、policy/hash 或 Run override 漂移 fail closed。confirmed pre-send release 按 ADR-0011 不消耗未发送 HTTP retry。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model 公平排序。 - typed audit、Run audit、task history/latency、Provider metadata、credential 非秘密事件和前端治理状态；P2-06 实现 SHA `9a20676…` 另交付固定低基数 Prometheus exporter/八条规则、canonical retention archive/verify/reconcile/restore/delete、Worker DB-time progress/liveness 聚合与全日志源治理。 @@ -206,6 +207,7 @@ | P2-03 Queue/Worker | 可靠基础已交付 | Redis Streams 通知、独立 Worker、数据库扫描、租约、心跳、fencing 和重复消息 no-op 已交付；P2-06 实现 SHA `9a20676…` 增加 generation 级 DB-time progress，dependency probe 仍只检查 capability | | P2-04 生命周期可靠性 | 可靠基础已交付 | retry/取消/恢复/dead-letter/终态重算及三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；外部调用仍不保证 exactly-once | | P2-05 并发治理 | 切片已实现 | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序和完整性 fail-closed 已实现；真实 PG/capacity/acceptance/精确 SHA CI 候选门禁已通过 | +| P2-05 observational overdraw 维护 | `in_progress` | ADR-0018 与 data-only `0007` 已进入工作树；只重算 overdrawn 并保留 ledger/actual，active reservation 时拒绝；本地完整门禁和当前库迁移已完成，commit/push/exact-SHA CI 待完成 | | P2-06 可观测性 | `completed` | exporter/八规则、retention CLI、Worker DB-time progress、`0005`/13 表 importer 与全日志源治理已实现；`9a20676…` 已 push、PR #3、实现 run `33164609388` 4/4，clean capacity/9/9 acceptance 全绿；evidence-doc commit `ec29596…` 已 push且精确 SHA run `33165775037` 4/4 | | P2-07 验证与运维 | `planned` | ADR-0016、独立计划与工作日志已建立；功能尚未实现，后续从最小只读 verifier 开始，再开展数据库+keyring restore、Redis 重建、告警响应与完整失败矩阵 | @@ -215,6 +217,7 @@ - `delivered`：真实 PostgreSQL 并发领取只有一个有效 lease；自然过期后由递增 fencing token 接管，陈旧 owner 写入被拒绝。 - `delivered/partial`：pending/running 取消有真实 Compose 证据；有限重试、超时和 dead-letter 有自动化状态机/Runner 证据，但尚未把所有失败组合都纳入完整生产式故障演练。 - `delivered`：历史 `0004` / 12 表 importer、四层治理、逐 attempt ledger、counter 重算 fail-closed、policy/override freeze、typed backpressure 和 finite fairness 已实现，并在精确候选 SHA 的真实 PG/capacity/acceptance 与远程 CI 通过。 +- `in_progress`：ADR-0018 修复不改变 protocol-v1 或历史事实；只有显式 input/output reservation 或由完整显式上界和价格派生的 reserved cost 才触发对应 overdraw，`0007` 只重算 materialized flag 且在 active reservation 时拒绝。本地完整验证已通过，但远程门禁尚未形成，不能标记完成。 - `completed`：P2-06 的 `0005` / 13 表 importer、Worker DB-time progress、固定低基数 exporter、八条规则、canonical audit archive/verify/reconcile/restore/delete 与全日志源治理已在 SHA `9a20676dcf545040782f04c166205d0043345753` 实现；本地 lint/test/integration/rules 与修复后 76-file 技术/安全终审全绿，实现已 push 到 PR #3，精确 SHA run `33164609388` 4/4 成功。Clean acceptance `.pytest_cache/artifacts/phase2-acceptance/llmbenchlab-p2-92e173eeee28/evidence.json`（SHA-256 `e4ffb8668fd3fa62d59b5d83f5c29eede35b327d88e6099345acd5950670fc47`）9/9、Worker `2/2/2/0/0`、cleanup C/V/N empty；clean capacity `.pytest_cache/artifacts/phase2-capacity/llmbenchlab-p2-ca5673061b0f/evidence.json`（SHA-256 `2382f9138f09028f269d76c341b236dd4089d678c8a2323582045fac2b4f5039`）记录 QPS `7.267474/12.962228/9.333604`、wall `8.255963/4.628834/6.428385s`、18/270/270/271/1230、0 question error/drift/duplicate/PEL/lag、expected=2/shortfall=0 与 cleanup C/V/N/image=0、image `1/1/0/0`。两者均为 clean-SHA Mock-only evidence，不是 SLO；此前 dirty evidence 保留为历史。Evidence-doc commit [`ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6) 已 push，其精确 SHA [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 4/4 成功，P2-06 仓库级收尾完成。 - `delivered`：`P2-local-control-plane-v2` 在 clean SHA `b6a35fe…` 完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 和 `qualified` 容量模型；只限定记录的 Mock-only 单机拓扑。 - `planned`：P2-07 工作包和恢复不变量已冻结，但功能尚未实现；后续开展 backup/restore、Redis 重建、告警响应与完整恢复矩阵。三个确定性 DB crash-seam 场景与正式恢复时长目标不能替代生产恢复认证。 @@ -241,7 +244,7 @@ ### 状态 -`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 的实现与 evidence-doc 精确 SHA CI 均已完成，状态为 `completed`；P2-07 工作包已建立，状态为 `planned`，功能尚未实现。不得把 Phase 2、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 +`in_progress`。可靠执行基础、P2-05 治理和 P2-01 v2 单机控制面资格已有可复核证据与精确 SHA 远程门禁；P2-06 的实现与 evidence-doc 精确 SHA CI 均已完成，状态为 `completed`。observational overdraw 维护已完成本地验证，但远程门禁待完成；P2-07 工作包已建立，状态为 `planned`，功能尚未实现。不得把 Phase 2、当前维护、生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once 标记为完成。 ## 6. Phase 3：标准 Benchmark 与代码评测 diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 9a6d58b..1b5045e 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -47,7 +47,7 @@ LLMBenchLab 当前开发基线是供个人开发者在受信任机器上使用 | 固定数据源/缓存漂移 | 下载被替换、缓存投毒或第三方题目误提交 | 固定 HTTPS revision、大小和 SHA-256；缓存拒绝 symlink；`artifacts/` Git 忽略 | Hash 不是发布者签名；HTTPS redirect 只有 scheme 检查，许可与来源仍需人工复核 | | XSS/内容注入 | 窃取页面数据或误导操作者 | React 默认文本转义；服务端不执行内容 | 未来引入 Markdown/HTML 时可能破坏边界；无 CSP | | 未授权访问 | 读写全部评测数据、启动付费 Run | 建议仅绑定本机；CORS 显式配置 | CORS 不是鉴权，非浏览器客户端不受其约束 | -| 费用滥用 | 大量、长输出或重复上游请求造成费用 | managed Run 将 policy/override 冻结进快照，在 global/provider/model/run 四层原子裁决 concurrency、fixed-minute RPM/TPM 与累计 request/Token/cost；backlog 有界、逐 HTTP retry 先 reserve/`send_started` 再 actual/conservative settlement；hard Token/cost 缺输入/输出上界或价格会在外发前 fail closed | 默认 policy 可关闭限制；`legacy_unmanaged` CLI 不在该边界；fixed-minute 允许边界突发；崩溃后的 Provider 幽灵请求、重复远端调用与 Provider 账单不能由本地 permit/ledger 消除；没有熔断，Provider 调用不是 exactly-once | +| 费用滥用 | 大量、长输出或重复上游请求造成费用 | managed Run 将 policy/override 冻结进快照，在 global/provider/model/run 四层原子裁决 concurrency、fixed-minute RPM/TPM 与累计 request/Token/cost；backlog 有界、逐 HTTP retry 先 reserve/`send_started` 再 actual/conservative settlement；hard Token/cost 缺显式输入/输出上界或价格会在外发前 fail closed；观测估算不冒充 hard input/cost reservation | 默认 policy 可关闭限制；`legacy_unmanaged` CLI 不在该边界；fixed-minute 允许边界突发；崩溃后的 Provider 幽灵请求、重复远端调用与 Provider 账单不能由本地 permit/ledger 消除；没有熔断，Provider 调用不是 exactly-once | | 数据丢失/篡改 | 证据不可复现或 stored Key 不可用 | 数据库约束、Hash、Run 快照、租约 fencing；credential AAD 绑定 Model/origin；typed audit event key/payload hash 幂等，operational/security 分级保留；PostgreSQL 是任务/治理事实来源 | 数据库/备份整体未加密；keyring 丢失不可恢复；应用 append-only 不是密码学 WORM、不可抵赖或数据库管理员防篡改，访问、归档与恢复仍由用户负责 | | 后台任务中断 | Run 不完整或重复执行 | 独立 Worker、数据库租约/心跳/fencing、有限重试、dead-letter、数据库对账 | 不构成生产 HA；外部 Provider 副作用不能由本地幂等约束去重 | | Redis 暴露或篡改 | 伪造/重放任务通知、拒绝服务、元数据泄漏 | Compose 不发布 Redis 宿主端口；消息不是事实来源且不含 Prompt/Key | 本地 Compose 未启用 Redis ACL/TLS；暴露到不可信网络会破坏可用性与保密性 | @@ -65,7 +65,7 @@ LLMBenchLab 当前开发基线是供个人开发者在受信任机器上使用 - 初始化前 `GET /api/v1/governance/policy` 是无副作用读取并返回 `404 governance_policy_not_initialized`；它不会把无鉴权的查询变成 policy 写入。`PUT` 必须提交全部 policy 字段，是不可变、内容寻址版本的 full-document apply，不是局部 PATCH。当前两个端点只适合可信 loopback；能够访问 `PUT` 的调用方可把限制设为 `null`（关闭）或 `0`（紧急拒绝），因此公开部署必须先加管理员授权。 - managed Run 创建在 global admission 锁内检查 backlog，并冻结全部 20 个 policy 字段、ID/hash、opaque provider scope、question quantum 与恰好四个 Run override（input reservation、request/Token/USD lifetime budget）。policy hash 与冻结 override 在 attempt 外发前重新校验，任何一侧漂移均 fail closed。四层锁序固定为 global→provider→model→run，网络期间不持有数据库锁。hard Token/TPM 需要显式 input reservation 与有限 `max_tokens`，hard cost 还需要 USD input/output price；缺失时在外发前 fail closed。治理 USD 公开上限为 `10000000.00000000`，Decimal 响应使用 JSON string；该上限使 SQLite IEEE-754 间距低于半个 `1e-8` 量化单位，PostgreSQL 则使用精确 `NUMERIC(20,8)`。 -- 每个 HTTP attempt 的 ledger 状态只允许 `reserved → send_started → settled_actual|settled_conservative`，或可证明未开始外发时 `reserved → released_pre_send`。明确 pre-send release 保留旧 ledger 并开启新 generation，只重试当前未发送 ordinal，不重置已经 `send_started` 的较小 ordinal。commit/usage/transport 不确定一律保守结算；这防止本地预算按零释放，却可能低估剩余额度。 +- 每个 HTTP attempt 的 ledger 状态只允许 `reserved → send_started → settled_actual|settled_conservative`，或可证明未开始外发时 `reserved → released_pre_send`。明确 pre-send release 保留旧 ledger 并开启新 generation，只重试当前未发送 ordinal，不重置已经 `send_started` 的较小 ordinal。commit/usage/transport 不确定一律保守结算；这防止本地预算按零释放，却可能低估剩余额度。没有显式 `input_token_reservation` 时，输入估算不写入新的 attempt hard reservation，也不参与 reserved cost 或 input/cost overdraw；Provider actual input 仍完整保存。显式 `max_tokens` 的 output reservation 独立生效。 - scope 和 UTC-minute 物化 counter 会在治理变更前从 never-delete ledger 重算；高、低漂移都停止新 admission/结算/对账，并由 API/Worker 边界用独立短事务尽力记录固定、不含损坏值的 `governance_integrity_error`。过期 lease takeover 在旧 ledger 校验失败时撤销新 owner 并使 Run fail closed，防止新 Worker 外发。 - 租约失效后 reconciler 会释放/结算本地 reservation，避免永久占用 admission。该动作无法停止已从 `send_started` 外发、仍在 Provider 运行的幽灵请求，也不证明崩溃后的真实远端并发仍受本地上限约束。本地 consumed 只用于 admission 与审计，必须另行核对 Provider 账单。 - typed audit payload 只接受 allowlist 短枚举/数值和 opaque ID，不含 Key、Authorization、URL、Prompt、原始回答或异常文本。事件重放以唯一 key 和 payload hash 检查一致性；默认 operational/security 至少保留 90/365 天。Retention CLI 会把过期完整事实写入权限收紧、canonical、内容/文件 hash 绑定的内部 JSONL，并支持离线 verify、精确 restore/delete/reconcile；hash 不是签名或 WORM，archive 仍含内部身份与时间线，必须按敏感运维文件保护。 @@ -289,7 +289,7 @@ Redis 仅可置于受控内部网络。当前本地 Compose 使用 AOF、无 ACL - 退出码 `3` 表示 `COMMIT` 已确认、但提交后验证或报告失败；导入已经提交。**禁止盲目重试或把它描述为回滚**，应先只读核验目标，必要时从已验证备份执行人工恢复。 - 工具是单向导入，不提供 PostgreSQL→SQLite 自动回迁。schema downgrade 也不等于数据平台回滚。 -`20260829_0006` 只补齐 canonical `0004` 的三个索引；兼容入口只接受 canonical schema，或缺失项为这三个索引的非空子集，以便 SQLite 非事务 DDL 中断后安全重入。新近成为 historical 的 PostgreSQL `0005` 也必须先通过 metadata drift 校验；任何额外差异仍 fail closed。SQLite preflight 与 repair migration 都会在恢复 single-active 唯一索引前拒绝多条 active policy。`0006 → 0005` 不删除对象；`20260828_0005 → 0004` 会移除 Worker progress，guard 在第一条 DDL 前拒绝任何 generation 行；`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata并拒绝任何相关事实。正常运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的旧备份并保留只读证据。只有隔离空数据库用于双方向往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 +当前 head `20260830_0007` 是 data-only 修复：它不改 schema、never-delete ledger、Provider actual usage、Response、audit 或 Run 终态，只依据冻结的 `evaluation_runs.input_token_reservation` 重算 `governance_scopes.overdrawn`。upgrade/downgrade 都会在任何更新前拒绝 `reserved/send_started` active reservation；downgrade 仅恢复旧派生谓词。`20260829_0006` 仍只补齐 canonical `0004` 的三个索引；兼容入口只接受 canonical schema，或缺失项为这三个索引的非空子集，以便 SQLite 非事务 DDL 中断后安全重入。新近成为 historical 的 PostgreSQL `0005/0006` 必须按各自规则通过 metadata drift 校验；任何额外差异仍 fail closed。SQLite preflight 与 repair migration 都会在恢复 single-active 唯一索引前拒绝多条 active policy。`0006 → 0005` 不删除对象；`20260828_0005 → 0004` 会移除 Worker progress，guard 在第一条 DDL 前拒绝任何 generation 行；`20260827_0004 → 0003` 会移除治理/ledger/audit/Provider metadata并拒绝任何相关事实。正常运行后的数据库不可原地 downgrade；应优先向前修复，或恢复经核验的旧备份并保留只读证据。只有隔离空数据库用于双方向往返验证；详见 [OPERATIONS.md](OPERATIONS.md)。 ## 10. 依赖与构建供应链 diff --git a/docs/TESTING.md b/docs/TESTING.md index 8093df5..075569a 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -218,6 +218,7 @@ OpenAI-compatible 测试只给进程内 transport 使用虚构 token；不得把 - API 只在数据库 commit 后 best-effort 发布通知；commit 失败不 publish，Redis 发布失败仍保留可由数据库 reconciliation 找回的 pending Run。 - policy GET 初始化前返回无副作用 `404 governance_policy_not_initialized`；PUT 要求全部 20 个字段，相同内容幂等、历史内容重激活原版本。Run admission 原子检查 backlog，并冻结 policy ID/version、完整 policy hash、opaque provider scope、question quantum，以及 `input_token_reservation` 和 lifetime request/Token/USD 四项 override；active attempt 必须使用这份 Run 快照，不能切换到后来激活的 policy。 - global/provider/model/run 四层按固定锁序共同裁决 concurrency/fixed-minute request+Token/lifetime budget；hard Token/TPM 缺 input reservation 或有限 `max_tokens`、hard cost 缺价格时在 Provider 外发前 fail closed。每次 reserve/send-start/finish/reconcile/lease renew 都从完整 ledger 重算四层 scope 与 minute bucket，任何高报、低报或缺 bucket 漂移都回滚并另写最小 `governance_integrity_error`，失败事务不留下部分 reservation/counter。 +- observational reservation 回归必须分别覆盖：Run 没有显式 input bound 时，attempt 的 `reserved_input_tokens/reserved_cost_usd` 为 `null`，actual input/cost 原样保存、四层 scope 不 overdraw 且下一次 reserve 可继续；显式 input、显式 output，以及由完整上界和冻结价格派生的 reserved cost 各自超额时仍把四层 scope 标记为 overdraw 并拒绝后续 admission。 - attempt ledger 覆盖 `reserved → send_started → settled_actual|settled_conservative` 与 retained `released_pre_send`。pre-send 重试递增 QuestionExecution generation 并保留当前尚未外发的 provider ordinal；已经到达 `send_started` 的较小 ordinal 仍被消费，不能因接管重置。HTTP retry、lease takeover、取消、重复投递和 commit-unknown 重放不 double-count，失租 `send_started` 只保守结算一次。 - Run 终态/defer/exhaust 等状态先提交，再 reconcile active ledger；reconcile 完整性失败必须保留已提交 Run 状态、向调用方失败并写独立 integrity event。过期租约接管若旧 ledger reconcile 失败，必须撤销新 lease 并把 Run 收敛到 failed/exhausted（有取消意图时为 cancelled），且不得外发 Provider 请求。 - backlog 满稳定 429；rate/concurrency defer 不生成 0 分 Response；每个 lease 只新增 frozen question quantum，cooperative yield 不增加 `failed_attempt_count`，最久未获服务 Run 优先再调度。 @@ -262,7 +263,7 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 `backend/tests/test_migrations.py` 使用独立临时 SQLite 和 Alembic 子进程验证： -- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260829_0006` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata、`worker_processes` 字段/约束、两个 bounded audit scan indexes，以及早期 `0004/0005` 三索引缺口、repair DDL 部分完成后的重入和 PostgreSQL `0005` metadata 白名单/额外 drift 拒绝控制流（Mock）。 +- 空库 upgrade/check/downgrade/upgrade 往返，以及 `20260830_0007` 最终 revision、可靠性/凭据/治理/ledger/audit/Provider metadata、`worker_processes` 字段/约束、两个 bounded audit scan indexes，以及早期 `0004/0005` 三索引缺口、repair DDL 部分完成后的重入和 PostgreSQL `0005/0006` metadata 白名单/额外 drift 拒绝控制流（Mock）。 - 有模型、Benchmark、题目、Run 与 Response 的 legacy schema 被一致性备份、严格识别并无损升级；题目按原插入顺序回填 0-based `position`。 - 与当前 metadata 一致但没有版本标记的库可安全收养，已有 head 重复 preflight 不生成多余备份。 - 部分表、server default/CHECK 内容或重名、PK/UNIQUE/FK/index/partial index、trigger、SQLite conflict policy/generated column、`STRICT`/`WITHOUT ROWID` 等未知 drift 在创建版本标记和备份前被拒绝；已在 head 的库同样验证。 @@ -271,6 +272,7 @@ SQLite 测试适合快速验证状态机和兼容路径；跨连接并发保证 - `0002 -> 0003` 会把旧 OpenAI-compatible Model 回填为 `environment`、Mock 回填为 `none`；只要凭据表有任意行，credential downgrade 在 DDL 前拒绝并保留二进制内容。 - `0003 -> 0004` 把既有 Run 标为 `legacy_unmanaged` 并保留 protocol-v1 证据；任意 policy/scope/bucket/question execution/attempt ledger/audit、新 Run fairness/governance 字段或 Response Provider metadata 存在时，`0004 -> 0003` 必须在第一条 DDL 前拒绝。只有隔离空库用于 `0004 -> 0003 -> 0004` roundtrip。 - `0004 -> 0005` 不回填虚构 Worker generation；任意 `worker_processes` 行都使 `0005 -> 0004` 在第一条 DDL 前拒绝。只有显式清空 process facts 或隔离空库才能往返，进入 `0004` 后原 governance/audit downgrade guard 继续生效。 +- `0006 -> 0007` 不改 schema 或历史 ledger/actual/Response，只重算 scope `overdrawn`：无显式 Run input bound 的 historical observational input/cost 超额被清除，显式 input/output 超额与由完整显式上界和价格派生的 cost 超额保留；`0007 -> 0006` 恢复旧派生结果。两个方向在任何更新前都拒绝 `reserved/send_started` active reservation。 - 应用启动 revision 门禁拒绝未迁移库；测试夹具中的 `create_all` 仅用于隔离临时库，并显式 stamp 到与 metadata 对应的 head，不是运行时建表路径。 目标化运行： @@ -280,7 +282,7 @@ cd backend uv run pytest tests/test_migrations.py ``` -真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline 并停止 API/Worker 后，从 head `0006` 发起 downgrade；`0006 -> 0005` 为 no-op，随后 populated `0005 -> 0004` 在任何有损 DDL 前拒绝。另建隔离空 PostgreSQL 跨过 `0005 -> 0004 -> 0005`，最终 `upgrade head` 回到 `0006` 并 check。历史 `0004 -> 0003` governance/audit guard 仍保留；schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 +真实 PostgreSQL `backend-integration` job 在空的专用 management database 上执行 migration 往返与 `alembic check`，验证 revision/DDL；它不提供已使用数据库可安全丢弃新事实的证明。带数据证据来自 Compose 验收：脚本完成 managed Mock baseline并停止 API/Worker 后，从 head `0007` 发起 downgrade；`0007 -> 0006` 只恢复旧 overdraw 派生谓词，`0006 -> 0005` 为 schema no-op，随后 populated `0005 -> 0004` 在任何有损 DDL 前拒绝。另建隔离空 PostgreSQL 跨过 `0005 -> 0004 -> 0005`，最终 `upgrade head` 回到 `0007` 并 check。历史 `0004 -> 0003` governance/audit guard 仍保留；schema downgrade 不是 PostgreSQL→SQLite 平台回迁。 ### 6.3 SQLite→PostgreSQL 导入 @@ -575,7 +577,7 @@ make phase2-acceptance 6. Redis 完全 stop/start；`live`/`health` 保持可用、`ready` 降级，API 仍以 `202` 提交数据库事实，Worker 仅靠 DB reconciliation 完成；Redis 恢复后新消息正常 ACK。 7. Worker 停止时取消 pending Run；Worker 恢复消费旧通知后终态和 0 Response 不漂移。 8. 运行中取消并再次 XADD 同一 Run；Response 数在取消后冻结，重复投递被 ACK 且 canonical snapshot 不变。 -9. 停止 API/Worker 后从 current head `20260829_0006` 尝试 downgrade 到 `0004`：schema-no-op `0006 -> 0005` 后，Worker progress rows 存在时必须在 `0005 -> 0004` 第一条有损 DDL 前拒绝，13 表计数、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0005 -> 0004 -> 0005`，最终回到 `0006` 并 check，随后重启 API/Worker。历史 `0004` governance/audit guard 继续由 migration 回归覆盖；schema downgrade 不是数据平台回迁。 +9. 停止 API/Worker 后从 current head `20260830_0007` 尝试 downgrade 到 `0004`：data-only `0007 -> 0006` 只按旧谓词重算 overdrawn，schema-no-op `0006 -> 0005` 后，Worker progress rows 存在时必须在 `0005 -> 0004` 第一条有损 DDL 前拒绝，13 表计数、Run/Response core protocol hash 与可靠性字段不变；另建独立空 PostgreSQL 完成 `0005 -> 0004 -> 0005`，最终回到 `0007` 并 check，随后重启 API/Worker。历史 `0004` governance/audit guard 继续由 migration 回归覆盖；schema downgrade 不是数据平台回迁。 任何一个场景失败、未运行、使用真实 Provider、最终 PEL/lag 非零或清理不完整，都不能把可靠执行基础写成通过。`--self-check-only` 只验证 Docker/Compose、隔离和清理 guard，不执行九场景，不能替代正式命令。精确 SHA `665244e…` 的最终本地运行已 9/9 通过；artifact 与 hash 见第 10.1 节。 diff --git a/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md index 8a4c327..0cd0c3f 100644 --- a/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md +++ b/docs/decisions/ADR-0015-observability-worker-progress-audit-retention.md @@ -5,7 +5,7 @@ - **Deciders**: LLMBenchLab maintainers - **Scope**: Phase 2 P2-06 metrics exporter、告警、Worker 主循环进展与 audit retention - **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的任务可观测性、[ADR-0009](ADR-0009-database-governance-audit-fair-scheduling.md) 的 typed audit 保留流程，以及 [ADR-0010](ADR-0010-phase-2-governance-delivery-boundaries.md) 的恢复指标边界 -- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将 schema-equivalent `0006` 加入 archive-v1 compatible-head allowlist +- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将 schema-equivalent `0006` 加入 archive-v1 compatible-head allowlist；[ADR-0018](ADR-0018-observational-token-estimates-are-not-hard-reservations.md) 将 data-only `0007` 加入同一 allowlist - **Preserves**: PostgreSQL/数据库事实来源、Redis 非权威通知、`llmbenchlab-protocol-v1`、write-only Provider Key、Mock-only 自动化和可信 loopback 部署边界 ## Context @@ -221,7 +221,7 @@ Dead-letter 与 integrity 是 15 分钟 rolling symptom；Prometheus 整个窗 - cutoff 不由用户任意传入；空集合也生成有效 archive。 - archive 默认绝不删除；运维顺序固定为 archive -> offline verify -> maintenance-window delete。 -单文件 canonical JSONL schema 为 `llmbenchlab-audit-archive-v1`：一行 header、零至一万行 `audit_event`、唯一末行 manifest。V1 的 event/payload/retention contract 与 compatible Alembic head allowlist 独立冻结；compatible heads 为 `20260828_0005` 与 schema-equivalent repair head `20260829_0006`。后者只恢复 canonical `0004` 的三个索引，不改变 archive event、字段或保留语义。`source_alembic_head` 不是装饰性字符串：write/verify/restore/delete/reconcile 都拒绝未列入 allowlist 的旧版、分支或未来 head；未来 schema 只有在证明 V1 全字段语义仍兼容后才能显式扩充 allowlist，否则必须提升 archive schema 并保留 V1 reader。Event 保存完整恢复事实： +单文件 canonical JSONL schema 为 `llmbenchlab-audit-archive-v1`：一行 header、零至一万行 `audit_event`、唯一末行 manifest。V1 的 event/payload/retention contract 与 compatible Alembic head allowlist 独立冻结；compatible heads 为 `20260828_0005`、schema-equivalent repair head `20260829_0006` 与 data-only repair head `20260830_0007`。`0006` 只恢复 canonical `0004` 的三个索引；`0007` 只按显式 hard reservation 语义重算 `governance_scopes.overdrawn`，两者都不改变 archive event、字段或保留语义。`source_alembic_head` 不是装饰性字符串：write/verify/restore/delete/reconcile 都拒绝未列入 allowlist 的旧版、分支或未来 head；未来 schema 只有在证明 V1 全字段语义仍兼容后才能显式扩充 allowlist，否则必须提升 archive schema 并保留 V1 reader。Event 保存完整恢复事实： ```text id, event_key, event_type, payload_hash, payload, retention_class, @@ -279,7 +279,7 @@ Delete 永不执行宽泛 `DELETE WHERE expires_at < cutoff`： ### 11. Schema、migration 与 importer -Worker progress revision 为 `20260828_0005`；当前 Alembic head `20260829_0006` 是 schema-equivalent index repair。两者都是 archive-v1 compatible source/target head： +Worker progress revision 为 `20260828_0005`，`20260829_0006` 是 schema-equivalent index repair，当前 Alembic head `20260830_0007` 是 observational-overdraw data repair。三者都是 archive-v1 compatible source/target head；`0007` 不新增表、字段、索引或 archive contract： - 创建 `worker_processes` 及本文约束/索引； - 为 audit archive 扫描增加 `(expires_at,id)` 索引； diff --git a/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md index 538eb49..e23363e 100644 --- a/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md +++ b/docs/decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md @@ -5,7 +5,7 @@ - **Deciders**: LLMBenchLab maintainers - **Scope**: Phase 2 P2-07 备份/恢复验证、Redis rebuild、告警响应与故障矩阵 - **Amends**: [ADR-0005](ADR-0005-durable-task-execution.md) 的恢复运维边界、[ADR-0007](ADR-0007-web-provider-credentials.md) 的数据库外 keyring 备份边界，以及 [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的告警与 archive 后续演练 -- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将尚未实施的 recovery-manifest-v1 exact head 显式更新为 schema-equivalent `20260829_0006` +- **Amended by**: [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 将尚未实施的 recovery-manifest-v1 exact head 显式更新为 schema-equivalent `20260829_0006`；[ADR-0018](ADR-0018-observational-token-estimates-are-not-hard-reservations.md) 再将 exact head 更新为 data-only `20260830_0007` - **Preserves**: PostgreSQL 唯一任务事实来源、Redis 非权威通知、write-only Provider Key、`llmbenchlab-protocol-v1`、Mock-only 自动化和可信本地运维边界 ## Context @@ -39,7 +39,7 @@ Phase 2 已交付 PostgreSQL 任务/治理事实、Redis Streams 通知、独立 - 创建备份前停止 admission、API、Worker、CLI mutation 和所有 application/audit writer；确认 Worker 已 graceful stop 或进入可解释 stale 状态。数据库健康探针可以继续只读连接，但任何未知 writer 都使备份集不合格。 - `pg_dump` 与 source manifest snapshot 必须来自同一停写窗口。恢复后的 13 表摘要与 manifest 不一致即证明期间发生漂移或 artifact 不匹配，整组 fail closed。 - 目标必须由操作者预先创建且没有任何用户 schema/table/Alembic row。先运行只读 empty-target 检查；目标非空时拒绝，绝不由工具清空。 -- 按 [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md)，恢复后数据库必须已经精确位于 Alembic `20260829_0006`。该 revision 与 `0005` 逻辑 schema 等价，只修复早期 `0004` 三索引缺口；P2-07 尚未实施，因此 recovery-manifest-v1 在实现前直接固定当前 head。禁止先升级旧 dump 再把它报告为原备份恢复成功；若未来支持新 head，必须在新 ADR/manifest schema 中显式声明兼容。 +- 按 [ADR-0017](ADR-0017-schema-equivalent-governance-index-repair.md) 与 [ADR-0018](ADR-0018-observational-token-estimates-are-not-hard-reservations.md)，恢复后数据库必须已经精确位于 Alembic `20260830_0007`。`0006` 与 `0005` 逻辑 schema 等价，只修复早期 `0004` 三索引缺口；`0007` 不改 schema 或 ledger/actual，只按“仅显式 input reservation 构成 hard bound”的规则重算 `governance_scopes.overdrawn`。P2-07 尚未实施，因此 recovery-manifest-v1 在实现前直接固定当前 head。禁止先升级旧 dump 再把它报告为原备份恢复成功；若未来支持新 head，必须在新 ADR/manifest schema 中显式声明兼容。 - 13 张核心表固定为： ```text @@ -72,7 +72,7 @@ Manifest schema 固定为 `llmbenchlab-recovery-manifest-v1`，UTF-8、canonical "schema": "llmbenchlab-recovery-manifest-v1", "backup_set_id": "00000000-0000-4000-8000-000000000000", "created_at_utc": "2026-08-30T00:00:00.000000Z", - "source_alembic_head": "20260829_0006", + "source_alembic_head": "20260830_0007", "source_git_commit": null, "database_dump": { "format": "postgresql-custom", diff --git a/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md b/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md index ac8965e..08339a5 100644 --- a/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md +++ b/docs/decisions/ADR-0017-schema-equivalent-governance-index-repair.md @@ -5,6 +5,7 @@ - **Deciders**: LLMBenchLab maintainers - **Scope**: 早期 `20260827_0004` 本地数据库兼容修复、audit archive head 兼容与 P2-07 recovery head - **Amends**: [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的 archive-v1 compatible-head allowlist，以及 [ADR-0016](ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) 的 P2-07 exact recovery head +- **Amended by**: [ADR-0018](ADR-0018-observational-token-estimates-are-not-hard-reservations.md) 将 data-only `0007` 加入 archive allowlist，并把尚未实施的 P2-07 exact recovery head 更新为 `20260830_0007` - **Preserves**: canonical 数据模型、13 表/importer、audit archive v1 字段语义、`llmbenchlab-protocol-v1`、未知 drift fail-closed 与数据库外 keyring ## Context @@ -34,6 +35,7 @@ - 当前应用启动仍只接受唯一 Alembic head。升级代码后必须先由唯一 migration owner 运行 `make migrate`；API/Worker 不自行迁移。 - 支持的 migration owner 入口始终先执行 preflight，再执行 Alembic。migration 内的 active-policy 与 reflection-visible 同名索引门禁是纵深保护，不替代 SQLite preflight 对 `DESC`/`COLLATE` 等深层 DDL modifier 的检查；裸 `alembic upgrade` 绕过 preflight 不属于受支持的旧库修复入口。 - 这不是新的产品 schema、API、Benchmark、评分协议、数据表或安全授权模型，也不推进 P2-07 功能实现。 +- 本 ADR 的 `0006` 索引修复事实与回滚边界保持不变；当前唯一 head 已由 ADR-0018 前进为 `0007`，不能把本 ADR 中“当前 `0006`”的历史措辞当作新的运行门禁。 ## Rejected alternatives diff --git a/docs/decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md b/docs/decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md new file mode 100644 index 0000000..ddea74a --- /dev/null +++ b/docs/decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md @@ -0,0 +1,50 @@ +# ADR-0018：观测 Token 估算不得作为治理硬预留 + +- **Status**: Accepted +- **Date**: 2026-08-30 +- **Deciders**: LLMBenchLab maintainers +- **Scope**: managed Run 的 Provider attempt reservation、overdraw 派生语义与历史物化修复 +- **Amends**: [ADR-0009](ADR-0009-database-governance-audit-fair-scheduling.md) 第 3、4 节的 input reservation / overdraw 语义 +- **Also amends**: [ADR-0015](ADR-0015-observability-worker-progress-audit-retention.md) 的 archive-v1 compatible-head allowlist，以及 [ADR-0016](ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md) 的 P2-07 exact recovery head +- **Preserves**: never-delete attempt ledger、Provider actual usage、显式 hard Token/cost fail-closed、四层 scope、`llmbenchlab-protocol-v1` + +## Context + +Runner 在 Run 未提供 `input_token_reservation` 时，用渲染消息 UTF-8 字节数除以四得到观测估算。该函数明确声明结果不是 hard bound，但实现仍把估算写入 `provider_call_reservations.reserved_input_tokens`，并在 Provider actual input usage 大于估算时把 global/provider/model/run 四层 scope 全部标记为 `overdrawn`。 + +真实 OpenCode Go `hy3` Run 暴露了该冲突：冻结 policy 和 Run override 没有任何 hard request/Token/cost 限制，第七个成功 attempt 的 input estimate/reservation 为 59，Provider actual 为 75。账本正确保存为 `settled_actual`，但下一题仍因共享 global scope 的 `overdrawn` 永久终止。前端又把所有 `overdrawn` 统一描述为“保守结算超额”，进一步误导为 Provider 额度或 conservative settlement 问题。 + +直接修改 actual usage、删除 ledger 或仅清除 materialized scope flag 都会破坏事实或在下一次 ledger 重算时失败；仅切换 policy 或新建 Run 也不能解除共享 global scope。 + +## Decision + +1. `input_token_reservation` 只有在 Run 明确提供该字段时才是 hard reservation。没有显式值时，UTF-8/tokenizer 估算最多用于观测，不写入 attempt 的 `reserved_input_tokens`，也不参与 reserved cost 或 overdraw 裁决。 +2. Provider actual input usage 始终原样保存。只有显式 input reservation 非空且 actual input 超过它时，input 维度才派生 `overdrawn`。由 input reservation 参与计算的 cost overdraw 同样要求显式 input reservation；显式 `max_tokens` 形成的 output reservation 继续独立参与 output overdraw。 +3. 新增数据 revision `20260830_0007`。它不改表结构或 never-delete ledger，只按修正后的派生规则重算 `governance_scopes.overdrawn`；downgrade 按旧规则重算，使旧应用不会遇到 materialized/ledger 漂移。upgrade/downgrade 都拒绝 active Provider reservation，要求唯一 migration owner 在停止 API/Worker 后执行。 +4. Runtime 的 scope ledger 重算使用同一规则：历史 managed reservation 通过关联 `evaluation_runs.input_token_reservation` 判断 input/cost reservation 是否显式；没有 Run 的内部 synthetic reservation 仍把调用者提供的 reservation 视为显式。 +5. 不增加数据库列或改写历史 reservation 数值。历史 estimate 59 与 actual 75 继续保留为审计事实，只是不再被解释成 hard overdraw。 +6. 前端 `overdrawn` 文案改为“实际用量曾被判定超过预留”，不再声称一定发生 conservative settlement；中性历史措辞也不会把升级前保留的误判终态描述成显式 hard reservation。 +7. `20260830_0007` 不改变 audit event/archive schema，因此加入 archive-v1 compatible-head allowlist；P2-07 尚未实施，其 exact recovery head 从 `0006` 修订为 `0007`。 + +## Consequences + +- 无限 policy 下不同 tokenizer/chat template 的正常估算偏差不再永久锁死 managed Provider 流量。 +- 启用 hard TPM/Token/cost 时仍要求显式 input reservation、有限 `max_tokens` 和适用价格；Provider actual 超过这些可证明边界时仍 fail closed。 +- 迁移保留所有历史 Response、actual usage、reservation、audit 与 Run 终态；已失败 Run 不改写为 completed，修复后需要新建 Run。 +- 当前数据库必须先停止 API/Worker、执行受支持的 migration 入口，再重启。直接 `UPDATE governance_scopes`、删除 reservation 或伪造 actual usage 均不受支持。 + +## Rejected alternatives + +- **提高 UTF-8 估算倍率**：任意通用倍率仍不是 Provider tokenizer/chat template 的可证明上界，只会延后同类误判。 +- **新建 Run 时设置更大 reservation 即可**：既有 global scope 已 overdrawn，新 admission 会先被共享 scope 拒绝。 +- **直接清除 scope flag**：materialized flag 会在下一次 ledger 校验时与旧派生规则冲突，且没有解释历史事实。 +- **删除或修改历史 ledger/actual usage**：破坏账单与审计证据，违反 never-delete 合同。 +- **增加 reservation provenance 列**：可以表达来源，但本次可由 Run 冻结的显式 override 无歧义推导；新增 schema 列会扩大迁移、importer 和 API 维护面。 + +## Validation + +- Unit：无显式 input reservation 时 actual 大于估算不 overdraw，账本 input reservation 与 reserved cost 为 `null`；显式 input/output reservation 超额，以及由完整显式上界与价格计算的 reserved cost 超额，仍 overdraw 并阻止后续 attempt。 +- Migration：SQLite 与 PostgreSQL 的 `0006 -> 0007` 对历史 observational input overdraw 清除 materialized flag但保留 ledger/actual；显式 reservation overdraw 保留；downgrade 恢复旧派生结果；active reservation 在更新前拒绝。 +- Runtime：scope materialization 从 never-delete ledger 按新规则重算且无 drift；现有本地数据库迁移后四层 overdrawn 清零，原 Run/7 Responses/407+599 Token 事实不变。 +- Frontend：Run detail 对 `*_overdrawn` 显示准确的新文案。 +- 全部自动化只使用 Mock/fixture，不调用真实 Provider。 diff --git a/docs/phases/PHASE-2-RELIABILITY.md b/docs/phases/PHASE-2-RELIABILITY.md index 79a3179..913f412 100644 --- a/docs/phases/PHASE-2-RELIABILITY.md +++ b/docs/phases/PHASE-2-RELIABILITY.md @@ -14,6 +14,7 @@ - 双 backlog 资格：[ADR-0014](../decisions/ADR-0014-dual-backlog-slo-profile.md) - 可观测性与审计保留：[ADR-0015](../decisions/ADR-0015-observability-worker-progress-audit-retention.md) - Schema-equivalent 索引修复：[ADR-0017](../decisions/ADR-0017-schema-equivalent-governance-index-repair.md) +- Observational reservation 修正：[ADR-0018](../decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) ## 阶段目标 @@ -21,11 +22,12 @@ ## 当前功能范围 -- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容；P2-06 implementation SHA `9a20676…` 将双方言 Alembic 链扩展至 `20260828_0005`，当前 schema-equivalent maintenance head `20260829_0006` 仅修复早期 `0004` 三索引缺口。 +- PostgreSQL 多 Worker 目标、SQLite 单 Worker 兼容；P2-06 implementation SHA `9a20676…` 将双方言 Alembic 链扩展至 `20260828_0005`，`20260829_0006` 仅修复早期 `0004` 三索引缺口；当前 data-only head `20260830_0007` 只按显式 hard reservation 语义重算 scope overdrawn。 - Redis at-least-once 通知；Run、取消、重试、租约、Response、终态、治理、attempt ledger 和 audit 全由数据库裁决。 - 原子 claim、数据库时间 lease/heartbeat、fencing、有限 retry/backoff、取消、过期接管、duplicate no-op 和 dead-letter。 - 停写只读 SQLite→空 PostgreSQL 的单向 importer；`0005` 按依赖顺序复制 13 张应用表并做 count/PK/content fingerprint，源有 live Worker generation 时拒绝，stopped/stale progress 可精确复制。keyring 仍在数据库之外。 - managed API Run 的 active policy ID/hash 与显式 input/Token/cost override 冻结；global/provider/model/run 四层 concurrency、固定分钟 RPM/TPM、global/run lifetime request/Token/USD budget。 +- 没有显式 `input_token_reservation` 时，观测 input 估算不再写成 hard reservation 或参与 reserved cost/input/cost overdraw；Provider actual usage 仍完整保存。显式 input、显式 `max_tokens` output，以及由完整上界和冻结价格派生的 reserved cost 超额继续 fail closed。 - 每个 Provider HTTP attempt 的 reserve→send-started→actual/conservative settlement 或 confirmed pre-send release never-delete ledger；materialized scope/bucket counter 仅作投影，任何高/低漂移 fail closed。 - 有限 backlog、typed `429`、database not-before、question quantum、dispatch/failure 分离和跨 Model due ordering。 - typed audit、分页 Run audit、task history counters、基于 Run 数据库时间戳的 queue/execution/end-to-end latency、严格规范化 Provider metadata 和非秘密 credential audit。 @@ -66,6 +68,7 @@ | P2-03 Queue/Worker | `foundation_delivered` | Redis 通知、DB scan、claim、lease/heartbeat/fencing、ACK/no-op 已交付；`9a20676…` 增加 generation 级 DB-time scan/claim/lease-heartbeat/progress 与 stale 聚合，dependency probe 仍只表示 capability | | P2-04 生命周期可靠性 | `foundation_delivered` | retry/backoff、取消、恢复、dead-letter、Response 幂等和三个确定性 DB crash-seam 场景已通过完整 Compose acceptance；Provider 外部副作用仍为 at-least-once | | P2-05 并发治理 | `slice_delivered` | 四层 concurrency/RPM/TPM/lifetime budget、per-attempt ledger、backpressure、finite quantum、公平排序、counter 重算 fail-closed 与 ADR-0011 已实现；精确 SHA 的真实 PG/capacity/acceptance/CI 候选门禁已通过 | +| P2-05 observational overdraw 维护 | `in_progress` | ADR-0018 与 data-only `0007` 已进入工作树；migration 只重算 overdrawn 并保留 ledger/actual/Response/Run，active reservation 时拒绝；本地完整验证和当前库迁移已完成，commit/push/exact-SHA CI 待完成 | | P2-06 可观测性 | `completed` | 固定 exporter/八规则、canonical retention CLI、Worker DB-time progress、`0005` / 13 表 importer、公共 retained-row 校验与全日志源治理已进入 clean commit `9a20676…`；clean capacity/9/9 acceptance、实现 CI 与 evidence-doc commit `ec29596…` 的 CI 4/4 均通过 | | P2-07 验证与运维 | `planned` | [ADR-0016](../decisions/ADR-0016-postgresql-keyring-recovery-and-redis-rebuild.md)、[独立计划](../plans/2026-08-28-phase-2-recovery-operations.md) 与 [工作日志](../worklogs/2026-08-28-phase-2-recovery-operations.md) 已建立；功能尚未实现，后续从最小只读 verifier 开始 | @@ -73,6 +76,8 @@ 2026-08-30 的个人本地维护已从最新非空 SQLite 一致性备份恢复 1 个 Mock Model、1 个 Demo Benchmark、15 Questions、1 个 completed Run 和 15 Responses，并让组合开发启动器把三服务详细输出分流到私有 Git 忽略日志。该操作有 staging 迁移、共有列摘要、完整性/外键/head、真实本地启动与 API/Web 读取证据，但不包含 PostgreSQL/keyring、Redis 或告警恢复认证，不能计作 P2-07 实施或改变本阶段状态。 +同日的 OpenCode Go `hy3` Run 又暴露 observational input estimate 被错误写成 hard reservation：7 个 attempt 全部 actual settlement，第七次 estimate/actual 为 59/75，而所有 hard policy/Run override 均为 `null`，四层 scope 却被标记 overdrawn。ADR-0018/`0007` 修正这一派生语义并把 UI 文案改为“实际用量曾被判定超过预留”；旧 ledger、actual usage、7 条 Response 和 failed/exhausted Run 终态不改写。本地完整门禁和当前 SQLite 迁移已通过，但 commit/push/exact-SHA CI 尚未完成，因此不能计为仓库级闭环。 + ## 验收标准与当前结论 - [x] 可靠执行基础：API/Worker restart、真实 lease-owner `SIGKILL`、Redis stop/start、duplicate delivery、pending/running cancel 和 lease takeover 有历史真实 PostgreSQL/Redis/Compose 证据。 @@ -85,6 +90,7 @@ - [x] **P2-01 完成**：clean SHA `b6a35fef1dd069ebb54b69955058915c722aa34d` 从零完成 1 warm-up + 5 measured、23/23 SLO、逐轮 hard invariant/cleanup 与 `qualified` 容量模型；aggregate SHA-256 `a76d167b…d0d9`。证据文档 commit `875f13a253c40b7573d45c6287385e60f2bb8f04` 已普通 push，[GitHub Actions run `33150080341`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33150080341) 对该精确 SHA 4/4 成功。结论只适用于固定 Mock 单机 profile。 - [x] **P2-06 仓库级闭环完成**：clean implementation commit `9a20676dcf545040782f04c166205d0043345753` 已 push，clean capacity/9/9 acceptance 与 [run `33164609388`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33164609388) 4/4 通过；evidence-doc commit `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6` 的 [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) 也精确 4/4 通过。 - [x] **0004 历史索引兼容修复完成**：schema-equivalent `20260829_0006`、仅允许三个已知索引缺失子集的可重入 SQLite preflight、PostgreSQL `0005` metadata 白名单控制流、重复 active/额外 drift 拒绝、真实失败备份副本升级及本地门禁均通过；实现 SHA [`8fb51b690ae6335b8ef93b3cbe54e039781fb173`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/commit/8fb51b690ae6335b8ef93b3cbe54e039781fb173) 的 [run `33263405214`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33263405214) 4/4 成功。historical PG missing-index 分支仍仅有 Mock 回归，标准 CI 真实 PG 只覆盖 fresh canonical 分支。 +- [ ] **Observational overdraw 修复尚未闭环**：目标行为与 `0007` migration 已通过本地完整测试、双方言 migration 和当前个人 SQLite 验真；commit/push 与 exact-SHA CI 证据待填。 - [ ] **P2-07 正式闭环未通过**：没有数据库+keyring backup/restore 认证、完整故障矩阵和告警处置演练。 ## 已实际运行的中间证据 @@ -116,6 +122,7 @@ | P2-06 evidence-doc remote gate | [run `33165775037`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33165775037) | 精确 `ec2959680459a14aa308bd4d9ebcc6bb7bfcf3a6`，四个必需 job 全 success；P2-06 仓库级闭环完成 | | 2026-08-29 DB compatibility repair | migration `52 passed`；完整 backend `927 passed, 33 skipped`、frontend `38 passed`；lint/smoke/config、真实失败备份副本与当前库 startup/check 全绿 | `8fb51b6…` 的 exact-SHA run `33263405214` 4/4；不改变 P2-07 planned 状态 | | 2026-08-30 本地恢复/静默启动维护 | 启动器 `3 passed`；完整 backend `930 passed, 33 skipped`、frontend `38 passed`；lint/build/smoke/config、SQLite digest/quick/FK/head、真实 API/Web 读取通过 | 个人本地 Demo 数据恢复和开发 UX；`5075bdb…` 的 run `33265171953` 4/4 成功，且不是 P2-07 恢复认证 | +| 2026-08-30 observational overdraw 修复 | backend `946 passed, 33 skipped`；真实 PG+Redis integration `33 passed`；双方言 migration 往返/check、`make lint`、frontend `39 passed`/build、Mock smoke `1 passed`、Compose config 与当前库迁移验真通过 | 当前 SQLite head `0007`，scope `4→0`，7 Responses/7 ledger/407 input/599 output、13 表行数、quick/FK 保持；无真实 Provider；远程门禁待完成 | 所有自动化模型行为只使用 Mock、MockTransport 或 stub；没有真实 Provider 或 API Key。 @@ -123,6 +130,7 @@ - importer 的 13 表依赖顺序包含原六张业务/凭据表、六张 governance/audit 表和 `worker_processes`；源必须停写且可读、不得有 live Worker generation，目标必须空，copy/对账在单一目标事务内完成。 - active policy/reservation、materialized/ledger 漂移和 schema fingerprint 不满足 preflight 时 fail closed。summary 只含 row count、PK/content digest，不输出业务内容或凭据材料。 +- `0007` upgrade/downgrade 均在任何 flag 更新前拒绝 active reservation；它只重算 `governance_scopes.overdrawn`，不得用直接 UPDATE、删除 reservation 或改小 Provider actual usage 代替。 - keyring 不随数据库复制；恢复目标必须另行安全取得匹配 keyring。数据库与 keyring 同时泄漏可恢复 Provider Key，丢失 keyring 则 stored credential 不可恢复。 - exit `2` 为提交前回滚；exit `4` 为 COMMIT 结果未知；exit `3` 为提交已确认但后验证/输出失败。exit 3/4 禁止盲目重试。 - downgrade `0005` 前必须停止 Worker、保存所需 progress 并显式清空 `worker_processes`；migration 不会静默丢弃 process facts。进入 `0004` 后，仍必须停止 API/Worker、关闭 admission、对账 active reservation 并满足原六类 governance/audit guard；不得为代码回滚删除已结算 ledger/audit。 @@ -141,8 +149,8 @@ ## 交付物与下一任务 -已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 与 evidence-doc SHA `ec29596…` 已分别通过精确 4/4 CI，仓库级闭环完成；P2-07 的 ADR/计划/日志已建立，后续按 [NEXT_TASK.md](../NEXT_TASK.md) 从最小只读 verifier 开始实施。 +已交付候选包括 `0004`、governance/audit 模型/repository、Adapter/Runner/Worker/API/UI、enhanced capacity/PG tests，以及 P2-01 v2 多轮资格。治理 SHA `665244e…` 已通过真实 integration/capacity/acceptance；SLO SHA `b6a35fe…` 已通过 23/23 本地资格与远程 4/4 CI。P2-06 implementation SHA `9a20676…` 与 evidence-doc SHA `ec29596…` 已分别通过精确 4/4 CI，仓库级闭环完成。ADR-0018 observational overdraw 维护的本地门禁与当前库迁移已完成，当前只剩远程闭环；之后才按 [NEXT_TASK.md](../NEXT_TASK.md) 开始 P2-07 最小只读 verifier。 ## 状态 -`in_progress`。P2-01 与 P2-06 已完成，P2-05 已交付；P2-07 状态为 `planned`，功能尚未实现，数据库+keyring backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 +`in_progress`。P2-01 与 P2-06 已完成，P2-05 主切片已交付；observational overdraw 维护已完成本地验证，但尚无 commit/push/exact-SHA CI。P2-07 状态为 `planned`，功能尚未实现，数据库+keyring backup/restore 和完整恢复演练仍缺失。不得把 Phase 2 或当前维护标为 `completed`，不得宣称生产 HA、灾难恢复 SLA、无限横向扩展或 Provider exactly-once。 diff --git a/docs/plans/2026-08-30-fix-observational-token-overdraw.md b/docs/plans/2026-08-30-fix-observational-token-overdraw.md new file mode 100644 index 0000000..2c9c87a --- /dev/null +++ b/docs/plans/2026-08-30-fix-observational-token-overdraw.md @@ -0,0 +1,73 @@ +# 修复观测 Token 估算误触发全局 overdraw 执行计划 + +- Owner: Codex +- Status: in_progress +- Created: 2026-08-30 +- Updated: 2026-08-30 +- Related phase: [Phase 2 — Reliability](../phases/PHASE-2-RELIABILITY.md) +- Worklog: [2026-08-30 工作日志](../worklogs/2026-08-30-fix-observational-token-overdraw.md) +- ADR: [ADR-0018](../decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) + +## Context + +OpenCode Go `hy3` 的 managed Run 在 15 题中成功完成 7 题后以 `governance_global_overdrawn` 终止。只读核查证明没有 429、HTTP retry 或 conservative settlement；第七次 Provider actual input 为 75，而本地非 hard UTF-8 估算为 59。所有 hard policy/Run override 均为 `null`，因此这是估算被错误当作硬预留的实现缺陷。 + +## Objective + +在不改写 Provider actual usage、历史 Response、attempt ledger 或评分协议的前提下，分离观测估算与显式 hard reservation，安全修复历史 materialized overdrawn，并让当前数据库重新接纳 managed Run。 + +## Scope + +- Runner/治理 repository 的 input/cost reservation 与 overdraw 派生修正。 +- 双方言 Alembic 数据迁移和安全回滚。 +- 当前 SQLite 的受控迁移与只读验证。 +- 前端 overdrawn 文案与目标回归测试。 +- README、API、Operations、Security、Testing、Phase/Status/Next Task/Changelog 同步。 + +## Non-goals + +- 不重跑或继续旧失败 Run，不发送真实 Provider 请求。 +- 不实现 OpenCode Responses/Anthropic Messages 协议。 +- 不增加 Tokenizer 依赖、Provider-specific 模型白名单或自动预算推荐。 +- 不删除/改写历史 ledger、actual usage、Response、audit 或 Run 终态。 + +## Assumptions + +- `evaluation_runs.input_token_reservation` 是历史 managed Run 是否显式冻结 input hard bound 的权威事实。 +- `reserved_output_tokens` 非空来自显式 `max_tokens`，继续是可执行的输出预留。 +- 当前 API/Worker 可在迁移窗口停止；数据库没有 active reservation。 + +## Implementation steps + +1. [completed] 只读定位 Run、policy、ledger、scope 和触发差异，接受 ADR-0018。 +2. [completed] 实现 runtime reservation/overdraw 语义和 `20260830_0007` 数据迁移，并通过 SQLite/真实 PostgreSQL 迁移与回归验证。 +3. [completed] 修正前端文案并补充 governance/runner/migration/frontend 回归；frontend `39 passed`。 +4. [completed] 停止本地服务，备份并迁移当前 SQLite；head=`0007`，四层误判 `4→0`，旧事实保持。 +5. [completed] 运行目标/完整测试、`make lint`、frontend build、Mock smoke、Compose config 与双方言 migration；完整结果见下表和工作日志。 +6. [in_progress] README/API/Architecture/Operations/Security/Testing/Deployment/Phase/Status/Next Task/Changelog/ADR 已同步新语义与 `0007`；最终日志结果、diff/秘密审查、commit/push 和精确 SHA CI 待完成。 + +## Risks + +| 风险 | 影响 | 缓解 | +|---|---|---| +| 清除真实 hard overdraw | 后续请求突破预算 | runtime/migration 都要求显式 Run input bound；output hard reservation保持独立 | +| 只改 flag 导致 ledger drift | 所有 admission fail closed | migration 与 runtime 共用同一可测试谓词语义，保留 ledger并重算 materialization | +| 活跃请求中迁移 | counter/结算竞态 | migration 拒绝 active reservation；本地先停止 API/Worker并复核 | +| 新 head 破坏 adoption/archive/acceptance | setup、归档或 CI 回归 | 更新 prepare-migration historical head、compatible allowlist、脚本常量和双方言测试 | +| 误发真实请求 | 产生费用/条款风险 | 自动化仅 Mock/fixture；本地验证只查 DB/API 状态，不创建 Run | + +## Validation + +| 验收项 | 命令或检查 | 预期结果 | 实际结果 | +|---|---|---|---| +| Governance unit | `pytest tests/test_governance.py` | 新旧 hard/observational 边界全通过 | 已纳入 backend 全量；backend `946 passed, 33 skipped` | +| Runner unit | 目标 reliability 测试 | 无显式 bound 时 ledger reservation 为 null | 已纳入 backend 全量通过 | +| Migration | SQLite + PostgreSQL migration tests | 0006→0007/回滚/guard 全通过 | SQLite 与真实 PostgreSQL upgrade/downgrade/upgrade/check 通过；真实 PostgreSQL+Redis integration `33 passed` | +| Frontend | Run detail test | 新文案通过 | frontend `39 passed`；lint/typecheck/build 通过，保留既有 662.40 kB chunk warning | +| Current DB | backup + migrate + SQL/API check | 4 scopes overdrawn→0，历史 7 Responses/usage 不变 | head=`0007`；scope `4→0`；7 Responses/7 ledger、407 input/599 output 保留；13 业务表行数一致；`quick_check=ok`、FK=0 | +| Full gates | `make test`、`make lint`、build、smoke、Compose config | 全部通过 | backend `946 passed, 33 skipped`；frontend `39 passed`；`make lint` exit 0；Mock smoke `1 passed`；frontend build 与 `docker compose config` 通过 | +| Remote | push + exact-SHA GitHub Actions | 必需 jobs 全绿 | 待执行 | + +## Rollback + +停止 API/Worker；保留 migration 前 SQLite 一致性备份。代码回滚时先在维护窗口执行 `0007 -> 0006`，它按旧 overdraw 谓词重算 flag，再运行旧应用。不得只替换代码、直接改 scope flag、删除 reservation 或把 actual usage 裁剪成 reservation。 diff --git a/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md b/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md new file mode 100644 index 0000000..670ae87 --- /dev/null +++ b/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md @@ -0,0 +1,90 @@ +# 2026-08-30 — 修复观测 Token 估算误触发全局 overdraw 工作日志 + +> 本日志记录实际发生的工作；所有命令以仓库根目录为基准。 + +## 元信息 + +- 日期：2026-08-30 +- 执行者：Codex +- 关联阶段：[Phase 2](../phases/PHASE-2-RELIABILITY.md) +- 关联计划：[执行计划](../plans/2026-08-30-fix-observational-token-overdraw.md) +- 关联 ADR：[ADR-0018](../decisions/ADR-0018-observational-token-estimates-are-not-hard-reservations.md) +- 当前状态：in_progress + +## 初始仓库状态 + +- 分支：`codex/complete-evaluation-workflow`，跟踪同名 origin 分支。 +- `git status --short --branch`：任务开始时工作树干净。 +- 已有用户改动：无 Git tracked 改动；本地 SQLite、keyring、日志与 dataset artifact 均保持为 Git 忽略资产。 +- 当前数据库 head：`20260829_0006`；开发 API/Worker/Web 正在运行。 + +## 目标与背景 + +修复 OpenCode-Hy3 Run 被本地治理误判为 global overdraw 的缺陷。冻结 policy 和 Run override 没有 hard 限制；第七次成功 Provider attempt 的观测 input estimate/reservation 为 59、actual 为 75，导致四层 scope 被永久标记。任务必须恢复后续 managed admission，同时保留历史实际用量、七条 Response 和失败 Run 终态。 + +## 范围 + +- ADR、runtime、数据迁移、历史 materialization、UI 文案、测试和强制文档。 +- 在无 active reservation 的维护窗口迁移当前个人 SQLite并只读验真。 + +## 非目标 + +- 不继续/改写旧失败 Run，不发送真实 Provider 请求。 +- 不实现其他 OpenCode 协议或扩大 Provider 功能。 +- 不清空数据库、ledger、audit 或治理计数。 + +## 验收标准 + +- [x] 无显式 input hard bound 时，Provider actual 大于观测 estimate 不触发 overdraw。 +- [x] 显式 input/output hard reservation 及由完整上界和价格派生的 reserved cost 超额仍 fail closed。 +- [x] 历史 0006 数据迁移保留 ledger/actual/Response，并按新语义重算 scope。 +- [x] 当前数据库四层误判解除，旧 Run 仍 failed/exhausted且 7 条 Response/407 input/599 output 保留。 +- [x] 前端不再把所有 overdrawn 描述为 conservative settlement。 +- [x] 目标/完整测试、`make lint`、frontend build、smoke、部署配置和双方言迁移验证通过。 +- [ ] 独立 commit 已 push，精确 SHA CI 必需 jobs 全部成功。 + +## 假设与风险 + +- 历史显式 input bound 可由冻结的 `evaluation_runs.input_token_reservation` 无歧义判断。 +- 迁移时若存在 active reservation 必须停止，不猜测结算结果。 +- 新 head 必须同步 setup/adoption、archive allowlist、acceptance 常量与 P2-07 exact-head 文档。 + +## 实施步骤 + +1. [completed] 读取仓库规则、Phase 2/治理文档，核对工作树和真实 Run/ledger。 +2. [completed] 建立 ADR-0018、执行计划与本日志。 +3. [completed] 实现代码、迁移与回归测试，并完成本地完整与真实基础设施验证。 +4. [completed] 执行当前数据库维护迁移和只读事实验证。 +5. [pending] 文档收尾、审查、commit/push/exact-SHA CI。 + +## 已确认事实 + +- Run `2181503c-eab2-4699-bede-db48bd078f95`：15 题完成 7 题，终态 `failed/exhausted`，reason=`governance_global_overdrawn`。 +- 7 个 attempt 全部 `settled_actual/succeeded`，无 conservative settlement、429 或 HTTP retry。 +- 第七次 input reservation/actual=`59/75`；其余六次 actual 未超过 reservation。 +- global/provider/model/run 四层 scope 均 active/reserved=0、consumed requests=7、input=407、output=599、overdrawn=true。 +- policy v1 和 Run overrides 的所有 hard request/Token/cost 限制均为 `null`。 + +## 实际修改、命令、结果与收尾 + +本次实际修改： + +- Runner 不再把非显式输入估算传给治理 repository；Run 未冻结 `input_token_reservation` 时，新 attempt 的 input reservation 与 reserved cost 为空，actual usage 仍原样保存。 +- Runtime、SQLite→PostgreSQL importer 和 Phase 2 capacity reconciliation 使用同一显式性规则：历史 managed reservation 关联 Run 冻结字段判断 input/cost，output reservation 独立，没有 Run 的 synthetic reservation 仍视调用者值为显式。 +- 新增 data-only `20260830_0007`；不改 schema/ledger/actual/Response/audit/Run，只重算 scope `overdrawn`，两个方向都在更新前拒绝 active reservation，downgrade 恢复旧谓词。 +- Migration adoption、archive-v1 compatible-head、P2-07 exact recovery head 和 acceptance/capacity 常量已同步到 `0007`；历史 `0006` 索引修复事实保留。 +- Run Detail 对 overdrawn 显示“实际用量曾被判定超过预留”，不再声称一定发生保守结算，也不会误述升级前保留的历史终态；对应目标回归已加入。 +- README、Changelog、Architecture、API、Operations、Security、Testing、Deployment、Project Status、Roadmap、Phase 2、Next Task、ADR/计划/日志已同步。 + +已实际完成的本地验证与数据维护： + +- Backend 全量：`946 passed, 33 skipped`；自动化仅使用 Mock/fixture。 +- `make lint` exit 0：backend Ruff check/format、frontend ESLint/TypeScript typecheck 全部通过。 +- 真实 PostgreSQL 16 + Redis 7 integration：`33 passed`；PostgreSQL migration upgrade/downgrade/upgrade/check 通过。 +- Frontend：`39 passed`；production build 通过；仅保留既有 662.40 kB chunk warning。 +- Mock smoke：`1 passed`；`docker compose config` 通过。 +- 当前个人 SQLite 已由一致性备份 `backend/data/llmbenchlab-pre-0007-20260830.db`（SHA-256 `3fcf7f4980e935a78a3d7b66351cb3556f6cbba5e37d348a82d9252d4d962792`）和 migration 入口自动备份共同保护后迁移到 `20260830_0007`；两份备份均收紧为 `0600`。四层 `overdrawn` scope 从 4 降为 0，旧 Run 仍为 `failed/exhausted`，7 Responses、7 ledger、407 input/599 output 全部保留，13 张业务表行数迁移前后一致，`quick_check=ok`、foreign-key violation=0。 +- 开发 API/Worker/Web 已重新启动；`/api/v1/health` 为 `ok`，任务 metrics 为 active Provider attempt `0`、overdrawn scope `0`、live Worker `1`。真实 PostgreSQL/Redis 回归使用的两个精确临时容器已停止并由 `--rm` 清除。 +- 未调用真实 Provider；尚未形成 commit/push/exact-SHA GitHub Actions 证据，因此任务仍为 `in_progress`，最后一项验收保持未勾选。 + +最终独立审查未发现 Blocker/High/Medium。剩余 Low 覆盖缺口是：seeded historical overdraw 与 active-reservation rollback 的长期自动回归目前在 SQLite，真实 PostgreSQL 已执行空库双方言 SQL 往返但未持久化同构数据 fixture；importer/capacity 的 provenance wiring 也分别由 helper/SQL 文本目标测试覆盖，尚无另一个端到端 historical row 场景。这些不改变当前修复结论，后续若扩充集成门禁可合并为一个串行 PostgreSQL fixture。 diff --git a/frontend/src/pages/RunDetailPage.tsx b/frontend/src/pages/RunDetailPage.tsx index 603fabd..6e51a8c 100644 --- a/frontend/src/pages/RunDetailPage.tsx +++ b/frontend/src/pages/RunDetailPage.tsx @@ -35,7 +35,7 @@ const governanceLimitLabels = { concurrency: "并发额度暂时占满", rpm: "每分钟请求额度暂时占满", tpm: "每分钟 Token 额度暂时占满", - overdrawn: "已发生保守结算超额", + overdrawn: "实际用量曾被判定超过预留", request_budget_exhausted: "累计请求硬预算已耗尽", token_budget_exhausted: "累计 Token 硬预算已耗尽", cost_budget_exhausted: "累计费用硬预算已耗尽", diff --git a/frontend/tests/run-detail-page.test.tsx b/frontend/tests/run-detail-page.test.tsx index 677d71f..d088cce 100644 --- a/frontend/tests/run-detail-page.test.tsx +++ b/frontend/tests/run-detail-page.test.tsx @@ -168,6 +168,21 @@ describe("RunDetailPage", () => { expect(screen.queryByText("governance_provider_rpm")).not.toBeInTheDocument(); }); + it("describes overdraw as actual usage exceeding an explicit reservation", async () => { + apiMocks.run.mockResolvedValue(runFixture({ + status: "failed", + governance_status: "exhausted", + governance_reason: "governance_global_overdrawn", + error_message: null, + })); + + renderPage(); + + expect(await screen.findByText("治理硬边界已终止 Run")).toBeInTheDocument(); + expect(screen.getByText(/全局实际用量曾被判定超过预留/)).toBeInTheDocument(); + expect(screen.queryByText(/保守结算超额/)).not.toBeInTheDocument(); + }); + it("explains exhausted and legacy-unmanaged boundaries without reflecting unknown reasons", async () => { const unsafeReason = "untrusted-reason-that-must-not-be-reflected"; apiMocks.run.mockResolvedValueOnce(runFixture({ diff --git a/scripts/phase2_acceptance.py b/scripts/phase2_acceptance.py index be3340f..e721061 100644 --- a/scripts/phase2_acceptance.py +++ b/scripts/phase2_acceptance.py @@ -43,7 +43,7 @@ PRE_GOVERNANCE_REVISION = "20260827_0003" GOVERNANCE_REVISION = "20260827_0004" WORKER_PROGRESS_REVISION = "20260828_0005" -DATABASE_HEAD_REVISION = "20260829_0006" +DATABASE_HEAD_REVISION = "20260830_0007" TASK_MESSAGE_VERSION = "llmbenchlab-run-task-v1" TASK_STREAM = "llmbenchlab:runs:v1" TASK_GROUP = "llmbenchlab-workers-v1" @@ -115,7 +115,6 @@ provider_scope=provider_scope_key("mock", None), lease_owner=owner, lease_token=lease.token, - estimated_input_tokens=8, reserved_output_tokens=64, reserved_cost_usd=Decimal("0"), ) diff --git a/scripts/phase2_capacity.py b/scripts/phase2_capacity.py index 60c792b..ad87225 100755 --- a/scripts/phase2_capacity.py +++ b/scripts/phase2_capacity.py @@ -247,8 +247,11 @@ def reconciliation_expectations( reservation.reserved_cost_usd, reservation.actual_input_tokens, reservation.actual_output_tokens, - reservation.actual_cost_usd + reservation.actual_cost_usd, + (reservation.run_id IS NULL OR run.input_token_reservation IS NOT NULL) + AS input_reservation_is_explicit FROM provider_call_reservations AS reservation + LEFT JOIN evaluation_runs AS run ON run.id = reservation.run_id CROSS JOIN LATERAL ( VALUES (reservation.global_scope_id), @@ -289,6 +292,8 @@ def reconciliation_expectations( state IN ('settled_actual', 'settled_conservative') AND ( ( + input_reservation_is_explicit + AND reserved_input_tokens IS NOT NULL AND actual_input_tokens IS NOT NULL AND actual_input_tokens > reserved_input_tokens @@ -299,6 +304,8 @@ def reconciliation_expectations( AND actual_output_tokens > reserved_output_tokens ) OR ( + input_reservation_is_explicit + AND reserved_cost_usd IS NOT NULL AND actual_cost_usd IS NOT NULL AND actual_cost_usd > reserved_cost_usd From cb00924ea3ba3d01ce5bc322b7eabdae1345baf3 Mon Sep 17 00:00:00 2001 From: gaoming <2168502171@qq.com> Date: Sun, 30 Aug 2026 03:31:34 +0800 Subject: [PATCH 12/12] test: accept nullable conservative reservation bounds --- CHANGELOG.md | 2 +- .../tests/test_phase2_acceptance_script.py | 40 +++++++++++++++++++ ...-08-30-fix-observational-token-overdraw.md | 5 ++- ...-08-30-fix-observational-token-overdraw.md | 6 ++- scripts/phase2_acceptance.py | 20 ++++++---- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1305fbe..177c14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ Phase 0 and the Phase 1 MVP vertical slice are complete. The Phase 2 reliable-ex ### Fixed -- Fixed managed Runs without an explicit `input_token_reservation` being permanently exhausted when Provider actual input exceeded a UTF-8 observational estimate. New attempts now leave input reservation and reserved cost unset in that case while preserving Provider actual usage; explicit input/output reservations and reserved cost derived from complete bounds plus frozen prices still enforce overdraw. Added data-only Alembic head `20260830_0007` to recompute only `governance_scopes.overdrawn`, preserve historical ledger/actual/Response/Run facts, reject upgrade or downgrade while any reservation is active, and restore the old derived predicate on downgrade. Run Detail now uses neutral historical wording for overdraw instead of attributing every case to conservative settlement. +- Fixed managed Runs without an explicit `input_token_reservation` being permanently exhausted when Provider actual input exceeded a UTF-8 observational estimate. New attempts now leave input reservation and reserved cost unset in that case while preserving Provider actual usage; explicit input/output reservations and reserved cost derived from complete bounds plus frozen prices still enforce overdraw. Added data-only Alembic head `20260830_0007` to recompute only `governance_scopes.overdrawn`, preserve historical ledger/actual/Response/Run facts, reject upgrade or downgrade while any reservation is active, and restore the old derived predicate on downgrade. Run Detail now uses neutral historical wording for overdraw instead of attributing every case to conservative settlement. The real-Compose acceptance seam now verifies nullable input/cost bounds without coercing a valid `null` settlement through `float()`. - Added ADR-0017 and the forward-only, schema-equivalent Alembic repair revision `20260829_0006` for databases that executed an early `0004` variant before three governance indexes were present. Migration preflight accepts canonical schemas or only a missing subset of those three indexes, so an interrupted SQLite repair is resumable; it backs up SQLite, validates newly historical PostgreSQL `0005` metadata, rejects multiple active policies, and continues to reject every other schema drift. - Prevented external `LogRecord` extras from injecting allowlisted request/run fields, rejected FIFO/non-regular archive inputs without blocking by opening them nonblocking before `fstat`, and made oversized audit archives fail on their global line cap before decoding attacker-controlled lines. - Made importer post-commit verification and no-op retention post-verification fail closed instead of treating an unchanged row count as sufficient evidence; PostgreSQL retention paths now preserve their advisory/row-lock gate through the mutation decision. diff --git a/backend/tests/test_phase2_acceptance_script.py b/backend/tests/test_phase2_acceptance_script.py index ad33e9e..ad2694e 100644 --- a/backend/tests/test_phase2_acceptance_script.py +++ b/backend/tests/test_phase2_acceptance_script.py @@ -339,6 +339,46 @@ def fake_psql(sql: str) -> subprocess.CompletedProcess[str]: assert result == {**projection, "sha256": script.canonical_hash(projection)} +@pytest.mark.parametrize( + "reservation", + [ + { + "reserved_input_tokens": None, + "reserved_output_tokens": 64, + "reserved_cost_usd": None, + "actual_input_tokens": None, + "actual_output_tokens": 64, + "actual_cost_usd": None, + }, + { + "reserved_input_tokens": 8, + "reserved_output_tokens": 64, + "reserved_cost_usd": 0.25, + "actual_input_tokens": 8, + "actual_output_tokens": 64, + "actual_cost_usd": 0.25, + }, + ], +) +def test_conservative_settlement_accepts_nullable_or_numeric_reserved_bounds( + reservation: dict[str, object], +) -> None: + assert script.conservative_settlement_matches_reserved_bounds(reservation) is True + + +def test_conservative_settlement_rejects_usage_that_differs_from_reserved_bounds() -> None: + reservation = { + "reserved_input_tokens": None, + "reserved_output_tokens": 64, + "reserved_cost_usd": None, + "actual_input_tokens": None, + "actual_output_tokens": 63, + "actual_cost_usd": None, + } + + assert script.conservative_settlement_matches_reserved_bounds(reservation) is False + + def test_run_all_includes_database_seams_before_outage_and_migration() -> None: harness = object.__new__(script.Phase2Acceptance) harness.evidence = {"status": "initializing"} diff --git a/docs/plans/2026-08-30-fix-observational-token-overdraw.md b/docs/plans/2026-08-30-fix-observational-token-overdraw.md index 2c9c87a..ec01f63 100644 --- a/docs/plans/2026-08-30-fix-observational-token-overdraw.md +++ b/docs/plans/2026-08-30-fix-observational-token-overdraw.md @@ -44,7 +44,7 @@ OpenCode Go `hy3` 的 managed Run 在 15 题中成功完成 7 题后以 `governa 3. [completed] 修正前端文案并补充 governance/runner/migration/frontend 回归；frontend `39 passed`。 4. [completed] 停止本地服务，备份并迁移当前 SQLite；head=`0007`，四层误判 `4→0`，旧事实保持。 5. [completed] 运行目标/完整测试、`make lint`、frontend build、Mock smoke、Compose config 与双方言 migration；完整结果见下表和工作日志。 -6. [in_progress] README/API/Architecture/Operations/Security/Testing/Deployment/Phase/Status/Next Task/Changelog/ADR 已同步新语义与 `0007`；最终日志结果、diff/秘密审查、commit/push 和精确 SHA CI 待完成。 +6. [in_progress] 实现 SHA `c6212db…` 已 push；首次精确 SHA CI 的 backend、PostgreSQL/Redis integration、frontend 三项成功，real-Compose job 暴露验收脚本对合法 nullable conservative settlement 执行 `float(None)`。最小兼容修正与本地 9/9 Compose 验收已通过，修正 commit/push 和新的精确 SHA CI 待完成。 ## Risks @@ -66,7 +66,8 @@ OpenCode Go `hy3` 的 managed Run 在 15 题中成功完成 7 题后以 `governa | Frontend | Run detail test | 新文案通过 | frontend `39 passed`；lint/typecheck/build 通过，保留既有 662.40 kB chunk warning | | Current DB | backup + migrate + SQL/API check | 4 scopes overdrawn→0，历史 7 Responses/usage 不变 | head=`0007`；scope `4→0`；7 Responses/7 ledger、407 input/599 output 保留；13 业务表行数一致；`quick_check=ok`、FK=0 | | Full gates | `make test`、`make lint`、build、smoke、Compose config | 全部通过 | backend `946 passed, 33 skipped`；frontend `39 passed`；`make lint` exit 0；Mock smoke `1 passed`；frontend build 与 `docker compose config` 通过 | -| Remote | push + exact-SHA GitHub Actions | 必需 jobs 全绿 | 待执行 | +| Compose acceptance correction | `make phase2-acceptance` | nullable hard-bound seam 与其余故障矩阵通过 | `9/9 passed`，隔离容器、卷和网络清理通过；目标脚本单测 `16 passed` | +| Remote | push + exact-SHA GitHub Actions | 必需 jobs 全绿 | `c6212db…` run `33270167616` 为 3/4；real-Compose 因 acceptance-only `float(None)` 失败，修正后的精确 SHA 待执行 | ## Rollback diff --git a/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md b/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md index 670ae87..adde28f 100644 --- a/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md +++ b/docs/worklogs/2026-08-30-fix-observational-token-overdraw.md @@ -55,7 +55,7 @@ 2. [completed] 建立 ADR-0018、执行计划与本日志。 3. [completed] 实现代码、迁移与回归测试，并完成本地完整与真实基础设施验证。 4. [completed] 执行当前数据库维护迁移和只读事实验证。 -5. [pending] 文档收尾、审查、commit/push/exact-SHA CI。 +5. [in_progress] 文档收尾、审查、修正 commit/push/exact-SHA CI。 ## 已确认事实 @@ -83,8 +83,10 @@ - 真实 PostgreSQL 16 + Redis 7 integration：`33 passed`；PostgreSQL migration upgrade/downgrade/upgrade/check 通过。 - Frontend：`39 passed`；production build 通过；仅保留既有 662.40 kB chunk warning。 - Mock smoke：`1 passed`；`docker compose config` 通过。 +- 首个实现 commit `c6212db2376ffc6b5f32c46b000aad8e7faf9b1f` 已普通 push。其精确 SHA GitHub Actions [run `33270167616`](https://github.com/CWNU-Open-Source-Community/LLMBenchLab/actions/runs/33270167616) 中 backend、真实 PostgreSQL/Redis integration、frontend 三个 job 成功；real-Compose job 在 `deterministic_database_seam_provider_and_response_commit_recovery` 失败。下载的 evidence 精确定位为 acceptance harness 对合法的 `actual_cost_usd=null` 执行 `float(None)`；生产治理和所有 Run 均正常完成。 +- 已将 conservative settlement 验收改为逐维精确比较 nullable/numeric reserved→actual，并加入 null、numeric、mismatch 回归。目标脚本测试 `16 passed`；完整 `make phase2-acceptance` 为 `9/9 passed`，此前失败场景通过，最终 cleanup 的 container/volume/network 均为空。第一次本地尝试在业务场景前被 Docker Hub 镜像代理 `docker.1panel.live` 的 403 阻断；随后仅使用临时匿名 Docker 配置与公共缓存补齐本机基础镜像，不修改全局 Docker 配置，实际验收成功。 - 当前个人 SQLite 已由一致性备份 `backend/data/llmbenchlab-pre-0007-20260830.db`（SHA-256 `3fcf7f4980e935a78a3d7b66351cb3556f6cbba5e37d348a82d9252d4d962792`）和 migration 入口自动备份共同保护后迁移到 `20260830_0007`；两份备份均收紧为 `0600`。四层 `overdrawn` scope 从 4 降为 0，旧 Run 仍为 `failed/exhausted`，7 Responses、7 ledger、407 input/599 output 全部保留，13 张业务表行数迁移前后一致，`quick_check=ok`、foreign-key violation=0。 - 开发 API/Worker/Web 已重新启动；`/api/v1/health` 为 `ok`，任务 metrics 为 active Provider attempt `0`、overdrawn scope `0`、live Worker `1`。真实 PostgreSQL/Redis 回归使用的两个精确临时容器已停止并由 `--rm` 清除。 -- 未调用真实 Provider；尚未形成 commit/push/exact-SHA GitHub Actions 证据，因此任务仍为 `in_progress`，最后一项验收保持未勾选。 +- 未调用真实 Provider。首个实现 SHA 的远程门禁如实保留为 3/4；nullable acceptance 修正尚待形成新 commit/push 并取得新的 exact-SHA 4/4 证据，因此任务仍为 `in_progress`，最后一项验收保持未勾选。 最终独立审查未发现 Blocker/High/Medium。剩余 Low 覆盖缺口是：seeded historical overdraw 与 active-reservation rollback 的长期自动回归目前在 SQLite，真实 PostgreSQL 已执行空库双方言 SQL 往返但未持久化同构数据 fixture；importer/capacity 的 provenance wiring 也分别由 helper/SQL 文本目标测试覆盖，尚无另一个端到端 historical row 场景。这些不改变当前修复结论，后续若扩充集成门禁可合并为一个串行 PostgreSQL fixture。 diff --git a/scripts/phase2_acceptance.py b/scripts/phase2_acceptance.py index e721061..4effb38 100644 --- a/scripts/phase2_acceptance.py +++ b/scripts/phase2_acceptance.py @@ -300,6 +300,17 @@ def canonical_hash(value: Any) -> str: return hashlib.sha256(encoded).hexdigest() +def conservative_settlement_matches_reserved_bounds( + reservation: dict[str, Any], +) -> bool: + """Return whether conservative settlement copied every configured bound.""" + + return all( + reservation[f"actual_{dimension}"] == reservation[f"reserved_{dimension}"] + for dimension in ("input_tokens", "output_tokens", "cost_usd") + ) + + def redact_text(value: str) -> str: redacted = value.replace(LOCAL_PASSWORD, "") redacted = re.sub( @@ -1959,14 +1970,7 @@ def injected_reservation(mode: str) -> Dict[str, Any]: send_started, ) self.require( - send_started["actual_input_tokens"] == send_started["reserved_input_tokens"] - and send_started["actual_output_tokens"] == send_started["reserved_output_tokens"] - and math.isclose( - float(send_started["actual_cost_usd"]), - float(send_started["reserved_cost_usd"]), - rel_tol=0, - abs_tol=1e-12, - ), + conservative_settlement_matches_reserved_bounds(send_started), "conservative settlement did not consume the reserved bounds", send_started, )